diff --git a/.gitattributes b/.gitattributes index 1c24fa411f027f33d17999f927770892165f7e93..6aa0fb0bf3f74411c27cf24664914eb2ed349de9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -94,3 +94,62 @@ kq00a7xd/episode_rosbags/episode_1_2025-01-07-18-02-10.bag filter=lfs diff=lfs m kq00a7xd/episode_rosbags/episode_0_2025-01-07-18-01-28.bag filter=lfs diff=lfs merge=lfs -text kq00a7xd/episode_rosbags/episode_4_2025-01-07-18-04-36.bag filter=lfs diff=lfs merge=lfs -text 2uvisx3r/episode_rosbags/episode_2_2025-01-07-20-36-41.bag filter=lfs diff=lfs merge=lfs -text +917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/episode_rosbags/episode_4_2025-01-07-17-58-51.bag filter=lfs diff=lfs merge=lfs -text +917c47lf/episode_rosbags/episode_4_2025-01-06-22-21-45.bag filter=lfs diff=lfs merge=lfs -text +2uvisx3r/episode_rosbags/episode_0_2025-01-07-20-34-59.bag filter=lfs diff=lfs merge=lfs -text +kq00a7xd/episode_rosbags/episode_2_2025-01-07-18-03-06.bag filter=lfs diff=lfs merge=lfs -text +kq00a7xd/episode_rosbags/episode_3_2025-01-07-18-03-47.bag filter=lfs diff=lfs merge=lfs -text +jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 filter=lfs diff=lfs merge=lfs -text +jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 filter=lfs diff=lfs merge=lfs -text +jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 filter=lfs diff=lfs merge=lfs -text +jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 filter=lfs diff=lfs merge=lfs -text +jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 filter=lfs diff=lfs merge=lfs -text +jwgj4azt/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +jwgj4azt/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +jwgj4azt/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +jwgj4azt/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +jwgj4azt/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/episode_rosbags/episode_3_2025-01-06-22-20-57.bag filter=lfs diff=lfs merge=lfs -text +917c47lf/episode_rosbags/episode_2_2025-01-06-22-19-54.bag filter=lfs diff=lfs merge=lfs -text +2uvisx3r/episode_rosbags/episode_3_2025-01-07-20-37-26.bag filter=lfs diff=lfs merge=lfs -text +e3yh61zm/episode_rosbags/episode_0_2025-01-07-17-55-52.bag filter=lfs diff=lfs merge=lfs -text +e3yh61zm/episode_rosbags/episode_3_2025-01-07-17-58-09.bag filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2zm0rh5c/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/episode_rosbags/episode_1_2025-01-07-20-35-51.bag filter=lfs diff=lfs merge=lfs -text +jwgj4azt/episode_rosbags/episode_1_2025-01-06-20-32-39.bag filter=lfs diff=lfs merge=lfs -text +jwgj4azt/episode_rosbags/episode_2_2025-01-06-20-33-19.bag filter=lfs diff=lfs merge=lfs -text +jwgj4azt/episode_rosbags/episode_3_2025-01-06-20-33-59.bag filter=lfs diff=lfs merge=lfs -text +jwgj4azt/episode_rosbags/episode_0_2025-01-06-20-32-00.bag filter=lfs diff=lfs merge=lfs -text +d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 filter=lfs diff=lfs merge=lfs -text +917c47lf/episode_rosbags/episode_1_2025-01-06-22-19-14.bag filter=lfs diff=lfs merge=lfs -text +d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +d4l0r92a/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +2uvisx3r/episode_rosbags/episode_4_2025-01-07-20-38-14.bag filter=lfs diff=lfs merge=lfs -text +jwgj4azt/episode_rosbags/episode_4_2025-01-06-20-34-40.bag filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/episode_rosbags/episode_1_2025-01-06-17-34-41.bag filter=lfs diff=lfs merge=lfs -text diff --git a/1exa8l2r/episode_rosbags/aligned_depth_to_color_K.npy b/1exa8l2r/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/1exa8l2r/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/1exa8l2r/episode_rosbags/cam_tf_world.npy b/1exa8l2r/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/1exa8l2r/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/1exa8l2r/episode_rosbags/color_K.npy b/1exa8l2r/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/1exa8l2r/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/1exa8l2r/episode_rosbags/depth_K.npy b/1exa8l2r/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/1exa8l2r/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/1exa8l2r/eval_robot.log b/1exa8l2r/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..72c7e1accac3b9fce7a41df45bed728761af79e2 --- /dev/null +++ b/1exa8l2r/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 16:58:43,670][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 16:58:43,673][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/1exa8l2r/wandb/debug-internal.log b/1exa8l2r/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..43777f567bf02b886650098dacd350052b3ab73f --- /dev/null +++ b/1exa8l2r/wandb/debug-internal.log @@ -0,0 +1,800 @@ +2025-01-07 16:58:49,655 INFO StreamThr :2192706 [internal.py:wandb_internal():85] W&B internal server running at pid: 2192706, started at: 2025-01-07 16:58:49.654614 +2025-01-07 16:58:49,655 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status +2025-01-07 16:58:49,668 INFO WriterThread:2192706 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/run-1exa8l2r.wandb +2025-01-07 16:58:49,668 DEBUG SenderThread:2192706 [sender.py:send():379] send: header +2025-01-07 16:58:49,675 DEBUG SenderThread:2192706 [sender.py:send():379] send: run +2025-01-07 16:58:49,962 INFO SenderThread:2192706 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files +2025-01-07 16:58:49,962 INFO SenderThread:2192706 [sender.py:_start_run_threads():1188] run started: 1exa8l2r with start time 1736287129.655871 +2025-01-07 16:58:49,978 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 16:58:49,979 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: check_version +2025-01-07 16:58:50,065 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 16:58:50,070 DEBUG HandlerThread:2192706 [system_info.py:__init__():26] System info init +2025-01-07 16:58:50,070 DEBUG HandlerThread:2192706 [system_info.py:__init__():41] System info init done +2025-01-07 16:58:50,070 INFO HandlerThread:2192706 [system_monitor.py:start():194] Starting system monitor +2025-01-07 16:58:50,070 INFO SystemMonitor:2192706 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 16:58:50,070 INFO HandlerThread:2192706 [system_monitor.py:probe():214] Collecting system info +2025-01-07 16:58:50,071 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 16:58:50,071 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started disk monitoring +2025-01-07 16:58:50,072 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 16:58:50,072 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started memory monitoring +2025-01-07 16:58:50,073 INFO SystemMonitor:2192706 [interfaces.py:start():188] Started network monitoring +2025-01-07 16:58:50,090 DEBUG HandlerThread:2192706 [system_info.py:probe():152] Probing system +2025-01-07 16:58:50,091 DEBUG HandlerThread:2192706 [system_info.py:_probe_git():137] Probing git +2025-01-07 16:58:50,096 DEBUG HandlerThread:2192706 [system_info.py:_probe_git():145] Probing git done +2025-01-07 16:58:50,096 DEBUG HandlerThread:2192706 [system_info.py:probe():200] Probing system done +2025-01-07 16:58:50,096 DEBUG HandlerThread:2192706 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:58:50.090144', 'startedAt': '2025-01-07T21:58:49.641407', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3934.3185000000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.953, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 723.8112297058105}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 16:58:50,096 INFO HandlerThread:2192706 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 16:58:50,096 INFO HandlerThread:2192706 [system_monitor.py:probe():227] Publishing system info +2025-01-07 16:58:50,096 DEBUG HandlerThread:2192706 [system_info.py:_save_code():44] Saving code +2025-01-07 16:58:50,102 DEBUG HandlerThread:2192706 [system_info.py:_save_code():67] Saving code done +2025-01-07 16:58:50,102 DEBUG HandlerThread:2192706 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 16:58:50,196 DEBUG HandlerThread:2192706 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 16:58:50,196 INFO HandlerThread:2192706 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 16:58:50,199 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 16:58:50,199 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 16:58:50,200 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 16:58:50,200 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 16:58:50,446 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/fx1h7118-wandb-metadata.json +2025-01-07 16:58:50,454 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 16:58:50,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: python_packages +2025-01-07 16:58:50,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:58:50,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:58:50,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:50,457 INFO HandlerThread:2192706 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/tb" + save: true +} +control { + relay_id: "8899518ba0484a498e6f13061035b9dd" +} +_info { + stream_id: "1exa8l2r" +} + +2025-01-07 16:58:50,457 DEBUG HandlerThread:2192706 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 16:58:50,472 INFO wandb-upload_1:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/m7e3sa56-code/FISH/eval_robot.py +2025-01-07 16:58:50,483 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/dthq6sum-diff.patch +2025-01-07 16:58:50,600 DEBUG SenderThread:2192706 [sender.py:send():379] send: telemetry +2025-01-07 16:58:50,600 DEBUG SenderThread:2192706 [sender.py:send():379] send: tbrecord +2025-01-07 16:58:50,600 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 16:58:50,600 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 with policy live +2025-01-07 16:58:50,780 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/xkk90gpj-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 16:58:50,963 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/diff.patch +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-metadata.json +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code/FISH/eval_robot.py +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/requirements.txt +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code/FISH +2025-01-07 16:58:50,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code +2025-01-07 16:58:51,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:52,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:52,963 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:58:53,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:54,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:55,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:58:55,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:56,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:57,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:57,825 DEBUG SenderThread:2192706 [sender.py:send():379] send: config +2025-01-07 16:58:58,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:58:58,964 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:58:59,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:00,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:00,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:00,965 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:01,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:02,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:02,966 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:03,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:04,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:04,966 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:05,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:59:05,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:59:05,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:05,611 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:06,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:06,967 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:07,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:08,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:09,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:10,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:11,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:11,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:12,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:13,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:14,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:14,969 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:15,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:16,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:16,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:16,970 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:17,461 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:18,461 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:18,970 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:19,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:20,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:59:20,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:59:20,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:21,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:21,592 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:21,971 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml +2025-01-07 16:59:22,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:23,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:24,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:25,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:26,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:27,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:27,689 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:28,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:28,973 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:29,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:30,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:31,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:32,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:33,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:33,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:34,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:35,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:59:35,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:35,498 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:59:36,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:36,981 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:37,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:38,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:38,602 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:39,464 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:40,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:41,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:42,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:43,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:44,200 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:44,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:45,152 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 16:59:45,152 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 with policy now +2025-01-07 16:59:45,152 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:59:45,153 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 16:59:45,153 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:59:45,154 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:59:45,404 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/1byh3xhq-media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 +2025-01-07 16:59:45,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:45,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 +2025-01-07 16:59:45,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 16:59:45,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media +2025-01-07 16:59:45,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos +2025-01-07 16:59:45,985 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 16:59:46,260 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/oo76nf8c-media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 +2025-01-07 16:59:46,465 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:46,984 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:47,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:48,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:49,409 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:49,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:50,073 DEBUG SystemMonitor:2192706 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 16:59:50,074 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 16:59:50,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:59:50,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:59:50,501 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:50,986 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:51,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:52,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:52,986 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 16:59:53,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:54,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:55,118 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:59:55,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:56,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:57,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:58,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:59:59,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:00,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:00,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:01,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:02,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:03,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:04,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:05,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:00:05,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:00:05,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:05,523 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:06,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:07,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:08,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:09,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:10,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:11,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:11,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:12,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:12,993 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:00:13,237 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/dsn5nkcp-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:00:13,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:14,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:14,994 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:15,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:16,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:16,994 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:17,222 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:17,470 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:18,470 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:18,995 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:19,470 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:20,075 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:00:20,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:00:20,456 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:00:20,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:21,470 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:22,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:22,574 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:23,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:24,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:25,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:26,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:27,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:28,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:28,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:29,471 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:30,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:31,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:32,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:33,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:33,797 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:34,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:35,000 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:35,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:00:35,455 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:00:35,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:36,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:37,001 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:37,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:38,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:39,002 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:39,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:39,797 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:40,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:41,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:42,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:43,003 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:43,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:44,385 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:00:44,385 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 with policy now +2025-01-07 17:00:44,388 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:00:44,388 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:00:44,388 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:00:44,395 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:00:44,474 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:44,639 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/lz0rmtpf-media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 +2025-01-07 17:00:45,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:00:45,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:45,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 +2025-01-07 17:00:45,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 17:00:45,280 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/sg1y8jde-media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 +2025-01-07 17:00:45,401 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:45,476 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:46,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:47,004 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:47,482 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:48,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:49,005 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:49,482 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:50,075 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:00:50,421 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:50,469 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:00:50,469 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:00:50,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:51,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:52,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:53,007 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:53,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:54,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:55,459 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:00:55,480 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:56,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:56,890 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:00:56,890 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:00:56,890 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:00:57,008 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:00:57,009 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:00:57,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:58,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:00:59,009 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:00:59,482 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:00,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:00,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:01,010 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:01,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:02,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:03,011 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:03,481 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:04,482 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:05,455 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:01:05,456 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:01:05,497 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:05,581 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:06,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:07,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:08,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:09,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:10,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:11,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:11,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:12,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:13,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:14,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:15,484 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:16,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:17,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:17,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:18,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:19,016 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:19,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:20,076 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:01:20,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:01:20,456 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:01:20,501 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:21,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:22,486 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:22,543 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:23,486 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:24,486 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:25,487 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:26,486 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:27,488 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:27,889 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:28,243 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:01:28,243 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 with policy now +2025-01-07 17:01:28,248 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:01:28,249 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:01:28,249 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:01:28,249 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:01:28,488 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:28,499 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/0g8ccs9p-media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 +2025-01-07 17:01:29,019 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:01:29,019 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:29,019 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 +2025-01-07 17:01:29,020 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 17:01:29,323 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/fh1frayt-media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 +2025-01-07 17:01:29,491 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:30,489 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:31,489 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:32,492 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:33,020 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:33,493 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:33,625 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:34,489 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:35,457 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:01:35,457 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:01:35,501 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:36,494 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:37,022 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:37,496 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:38,500 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:39,495 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:39,534 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:40,499 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:41,499 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:42,496 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:43,499 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:44,500 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:45,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:45,500 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:46,499 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:47,501 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:47,747 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:01:47,747 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:01:47,747 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:01:48,025 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:01:48,026 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:01:48,285 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/j7murppu-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:01:48,506 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:49,026 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:49,505 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:50,076 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:01:50,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:01:50,458 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:01:50,506 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:50,537 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:51,027 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:51,612 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:52,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:53,027 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:01:53,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:54,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:55,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:56,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:01:56,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:57,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:58,598 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:01:59,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:00,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:01,599 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:02,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:02,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:03,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:04,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:05,456 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:02:05,457 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:02:05,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:06,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:07,594 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:07,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:08,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:09,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:10,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:11,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:12,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:13,467 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:13,600 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:14,601 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:15,601 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:16,601 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:17,601 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:18,577 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:18,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:19,604 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:19,713 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:02:19,713 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 with policy now +2025-01-07 17:02:19,716 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:02:19,716 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:02:19,716 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:02:19,717 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:02:19,972 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/bmos0zwk-media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 +2025-01-07 17:02:20,036 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:02:20,036 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 +2025-01-07 17:02:20,037 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 17:02:20,077 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:02:20,298 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/r6f9hx37-media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 +2025-01-07 17:02:20,461 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:02:20,461 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:02:20,609 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:21,037 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:21,609 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:22,613 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:23,607 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:24,285 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:24,610 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:25,038 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:25,612 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:26,611 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:27,039 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:27,613 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:28,612 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:29,612 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:30,066 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:30,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:31,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:32,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:32,800 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:02:32,801 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:02:32,802 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:02:33,041 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:02:33,041 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:02:33,041 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:33,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:34,621 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:35,041 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:35,349 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:35,466 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:02:35,466 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:02:35,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:36,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:37,042 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:37,615 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:38,615 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:39,043 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:02:39,615 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:40,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:40,615 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:41,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:42,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:43,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:44,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:45,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:45,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:46,616 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:47,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:48,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:49,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:50,077 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:02:50,458 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:02:50,458 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:02:50,593 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:50,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:51,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:52,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:53,617 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:54,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:55,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:56,475 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:02:56,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:57,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:58,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:02:59,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:00,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:01,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:02,473 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:02,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:03,618 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:04,619 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:05,468 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:03:05,468 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:03:05,628 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:05,879 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:03:05,879 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 with policy now +2025-01-07 17:03:05,880 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:03:05,880 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:05,880 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:05,881 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:06,062 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:06,062 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 +2025-01-07 17:03:06,062 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval +2025-01-07 17:03:06,138 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/6ccuuatk-media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 +2025-01-07 17:03:06,395 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/ii264243-media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 +2025-01-07 17:03:06,632 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:07,062 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:07,637 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:07,917 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:08,636 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:09,636 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:10,636 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:11,063 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:11,639 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:12,638 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:13,064 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:13,341 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:13,637 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:14,643 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:15,648 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:16,645 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:17,638 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:18,485 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:18,655 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:19,653 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:20,082 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:03:20,463 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:03:20,463 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:03:20,658 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:21,650 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:22,655 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:23,635 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:23,661 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:24,661 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:25,653 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:26,660 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:27,656 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:28,658 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:29,495 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:29,661 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:30,657 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:31,661 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:32,674 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:33,669 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:33,873 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:33,874 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:33,874 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:34,071 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:34,071 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:34,284 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:03:34,284 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:03:34,322 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/qnep3tbh-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:34,462 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:03:34,665 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:34,779 INFO wandb-upload_0:2192706 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpdfbhba_e +2025-01-07 17:03:34,845 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 17:03:34,853 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table +2025-01-07 17:03:35,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval +2025-01-07 17:03:35,243 INFO SenderThread:2192706 [sender.py:send_request_log_artifact():1525] logged artifact run-1exa8l2r-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDQ4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY4MzE2Nw==', 'latestArtifact': None}} +2025-01-07 17:03:35,243 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:03:35,243 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json with policy now +2025-01-07 17:03:35,243 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:35,243 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:35,244 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,244 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,244 DEBUG SenderThread:2192706 [sender.py:send():379] send: telemetry +2025-01-07 17:03:35,244 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,248 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,248 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,249 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,249 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,249 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,249 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,250 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,250 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,250 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,251 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,251 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,251 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,251 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,252 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,252 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,252 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,252 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,252 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,253 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,253 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:35,253 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:35,253 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 17:03:35,442 INFO wandb-upload_1:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/rnwxrymp-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 17:03:35,453 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/pw46t55s-media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json +2025-01-07 17:03:35,460 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 17:03:35,665 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:35,689 INFO wandb-upload_1:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/idgarda9-media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 17:03:35,795 INFO wandb-upload_0:2192706 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_j60bioa +2025-01-07 17:03:36,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:36,181 INFO SenderThread:2192706 [sender.py:send_request_log_artifact():1525] logged artifact run-1exa8l2r-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDUxNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY4MzE2OQ==', 'latestArtifact': None}} +2025-01-07 17:03:36,182 DEBUG SenderThread:2192706 [sender.py:send():379] send: files +2025-01-07 17:03:36,182 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json with policy now +2025-01-07 17:03:36,182 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:36,183 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:36,183 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:36,183 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: stop_status +2025-01-07 17:03:36,427 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/u9ans44y-media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json +2025-01-07 17:03:36,537 DEBUG SenderThread:2192706 [sender.py:send():379] send: exit +2025-01-07 17:03:36,537 INFO SenderThread:2192706 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 17:03:36,538 INFO SenderThread:2192706 [sender.py:send_exit():588] handling runtime: 286 +2025-01-07 17:03:36,538 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:36,538 INFO SenderThread:2192706 [sender.py:send_exit():594] send defer +2025-01-07 17:03:36,538 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,538 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 17:03:36,538 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:36,538 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 17:03:36,539 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 1 +2025-01-07 17:03:36,539 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,539 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 17:03:36,539 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:36,539 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 17:03:36,539 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 2 +2025-01-07 17:03:36,539 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,539 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 17:03:36,539 INFO HandlerThread:2192706 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 17:03:36,539 DEBUG SystemMonitor:2192706 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 17:03:36,539 DEBUG SystemMonitor:2192706 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 17:03:36,540 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 17:03:36,540 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 17:03:36,593 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 17:03:36,593 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 17:03:36,593 INFO HandlerThread:2192706 [interfaces.py:finish():200] Joined network monitor +2025-01-07 17:03:36,594 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:36,594 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 17:03:36,594 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 3 +2025-01-07 17:03:36,594 DEBUG SenderThread:2192706 [sender.py:send():379] send: stats +2025-01-07 17:03:36,595 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,595 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 17:03:36,595 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:36,595 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 17:03:36,595 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 4 +2025-01-07 17:03:36,595 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:36,595 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 17:03:37,072 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:37,073 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:39,073 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:42,880 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:42,881 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:42,881 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 17:03:42,881 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:42,882 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 17:03:42,882 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 5 +2025-01-07 17:03:42,882 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:42,882 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:42,883 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 17:03:42,883 DEBUG SenderThread:2192706 [sender.py:send():379] send: history +2025-01-07 17:03:42,883 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: summary_record +2025-01-07 17:03:42,885 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:42,885 DEBUG SenderThread:2192706 [sender.py:send():379] send: summary +2025-01-07 17:03:42,885 INFO SenderThread:2192706 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 17:03:42,885 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:42,885 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 17:03:42,885 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 6 +2025-01-07 17:03:42,885 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:42,886 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 17:03:42,886 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:42,886 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 17:03:42,889 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 17:03:43,012 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 7 +2025-01-07 17:03:43,013 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:43,013 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 17:03:43,013 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:43,013 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 17:03:43,075 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:43,075 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml +2025-01-07 17:03:43,539 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:46,471 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 8 +2025-01-07 17:03:46,471 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:46,472 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:46,472 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 17:03:46,472 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:46,472 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 17:03:46,472 INFO SenderThread:2192706 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 17:03:46,474 INFO SenderThread:2192706 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 17:03:46,476 INFO SenderThread:2192706 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 17:03:46,490 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 9 +2025-01-07 17:03:46,490 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:46,490 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 17:03:46,490 DEBUG SenderThread:2192706 [sender.py:send():379] send: artifact +2025-01-07 17:03:46,539 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:46,995 INFO wandb-upload_1:2192706 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpn5jjzskl +2025-01-07 17:03:46,995 INFO wandb-upload_2:2192706 [upload_job.py:push():85] Skipped uploading /tmp/tmpfj24jvix/wandb-job.json +2025-01-07 17:03:47,077 INFO Thread-12 :2192706 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:47,175 INFO wandb-upload_0:2192706 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpr5j4j1ba +2025-01-07 17:03:47,710 INFO SenderThread:2192706 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTgyOTA5NQ==', 'versionIndex': 433}}} +2025-01-07 17:03:47,710 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:47,710 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 17:03:47,710 INFO SenderThread:2192706 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 17:03:48,077 INFO SenderThread:2192706 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files +2025-01-07 17:03:48,077 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml config.yaml +2025-01-07 17:03:48,078 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json wandb-summary.json +2025-01-07 17:03:48,078 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:48,078 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/requirements.txt requirements.txt +2025-01-07 17:03:48,081 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-metadata.json wandb-metadata.json +2025-01-07 17:03:48,081 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log output.log +2025-01-07 17:03:48,081 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/diff.patch diff.patch +2025-01-07 17:03:48,081 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 media/videos/eval/0_eval_0_133b41ccfd01da169e6f.mp4 +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 media/videos/eval/3_eval_3_dc74b0d17439e756067e.mp4 +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 media/videos/eval/1_eval_1_46ad08891ecc2d017fe5.mp4 +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 media/videos/eval/4_eval_4_e22e33fe5bb8bbb0d7c4.mp4 +2025-01-07 17:03:48,082 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 media/videos/eval/2_eval_2_2b1402d0e72f1d58d0bd.mp4 +2025-01-07 17:03:48,084 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json media/table/eval/success_rate_ci_5_85c54431b4eb7c8145a7.table.json +2025-01-07 17:03:48,084 INFO SenderThread:2192706 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json media/table/eval/total_success_rate_ci_6_a80255a63fbf0577db61.table.json +2025-01-07 17:03:48,084 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 10 +2025-01-07 17:03:48,084 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:48,084 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,084 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 17:03:48,087 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,087 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 17:03:48,087 INFO SenderThread:2192706 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:03:48,268 INFO wandb-upload_1:2192706 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/config.yaml +2025-01-07 17:03:48,276 INFO wandb-upload_2:2192706 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/wandb-summary.json +2025-01-07 17:03:48,289 INFO wandb-upload_0:2192706 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/requirements.txt +2025-01-07 17:03:48,292 INFO wandb-upload_4:2192706 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/files/output.log +2025-01-07 17:03:48,342 INFO wandb-upload_3:2192706 [upload_job.py:push():130] Uploaded file /tmp/tmprohsv1cgwandb/yh7a92kv-events.out.tfevents.1736287130.leonmkim-ROG-Strix-G15CS-G15CS.2192541.0 +2025-01-07 17:03:48,540 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:48,541 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:48,542 INFO Thread-11 (_thread_body):2192706 [sender.py:transition_state():614] send defer: 11 +2025-01-07 17:03:48,542 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,542 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 17:03:48,543 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,543 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 17:03:48,543 INFO SenderThread:2192706 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 17:03:48,544 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 12 +2025-01-07 17:03:48,544 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,544 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 17:03:48,544 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,544 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 17:03:48,544 INFO SenderThread:2192706 [file_stream.py:finish():601] file stream finish called +2025-01-07 17:03:48,711 INFO SenderThread:2192706 [file_stream.py:finish():605] file stream finish is done +2025-01-07 17:03:48,711 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 13 +2025-01-07 17:03:48,711 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,711 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 17:03:48,712 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,712 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 17:03:48,712 INFO SenderThread:2192706 [sender.py:transition_state():614] send defer: 14 +2025-01-07 17:03:48,712 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: defer +2025-01-07 17:03:48,712 INFO HandlerThread:2192706 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 17:03:48,712 DEBUG SenderThread:2192706 [sender.py:send():379] send: final +2025-01-07 17:03:48,712 DEBUG SenderThread:2192706 [sender.py:send():379] send: footer +2025-01-07 17:03:48,712 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: defer +2025-01-07 17:03:48,712 INFO SenderThread:2192706 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 17:03:48,713 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:48,713 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 17:03:48,713 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 17:03:48,713 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:48,713 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: server_info +2025-01-07 17:03:48,715 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 17:03:48,715 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 17:03:48,716 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 17:03:48,769 DEBUG SenderThread:2192706 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 17:03:48,769 INFO MainThread:2192706 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 17:03:48,770 INFO MainThread:2192706 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 17:03:48,770 INFO MainThread:2192706 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 17:03:48,770 DEBUG HandlerThread:2192706 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 17:03:48,770 INFO HandlerThread:2192706 [handler.py:finish():882] shutting down handler +2025-01-07 17:03:49,713 INFO WriterThread:2192706 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/run-1exa8l2r.wandb +2025-01-07 17:03:49,769 INFO SenderThread:2192706 [sender.py:finish():1615] shutting down sender +2025-01-07 17:03:49,769 INFO SenderThread:2192706 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 17:03:49,769 INFO SenderThread:2192706 [file_pusher.py:join():175] waiting for file pusher diff --git a/1exa8l2r/wandb/debug.log b/1exa8l2r/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2825e6008a3b55f92981e89475abb7e504d97e92 --- /dev/null +++ b/1exa8l2r/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 16:58:49,645 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Configure stats pid to 2192541 +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/settings +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug.log +2025-01-07 16:58:49,646 INFO MainThread:2192541 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/logs/debug-internal.log +2025-01-07 16:58:49,647 INFO MainThread:2192541 [wandb_init.py:init():569] calling init triggers +2025-01-07 16:58:49,647 INFO MainThread:2192541 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r'} +2025-01-07 16:58:49,647 INFO MainThread:2192541 [wandb_init.py:init():619] starting backend +2025-01-07 16:58:49,648 INFO MainThread:2192541 [wandb_init.py:init():623] setting up manager +2025-01-07 16:58:49,653 INFO MainThread:2192541 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 16:58:49,655 INFO MainThread:2192541 [wandb_init.py:init():631] backend started and connected +2025-01-07 16:58:49,667 INFO MainThread:2192541 [wandb_init.py:init():720] updated telemetry +2025-01-07 16:58:49,673 INFO MainThread:2192541 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 16:58:49,978 INFO MainThread:2192541 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 16:58:50,016 INFO MainThread:2192541 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 16:58:50,017 INFO MainThread:2192541 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 16:58:50,455 INFO MainThread:2192541 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 16:58:50,455 INFO MainThread:2192541 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 16:58:50,455 INFO MainThread:2192541 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 16:58:50,455 INFO MainThread:2192541 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 16:58:50,457 INFO MainThread:2192541 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 16:58:50,457 INFO MainThread:2192541 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/1exa8l2r/tb, True +2025-01-07 16:58:57,824 INFO MainThread:2192541 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 17:03:49,771 WARNING MsgRouterThr:2192541 [router.py:message_loop():77] message_loop has been closed diff --git a/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/run-1exa8l2r.wandb b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/run-1exa8l2r.wandb new file mode 100644 index 0000000000000000000000000000000000000000..193e563ac7556c35e64a7f3360d7502cc1324923 Binary files /dev/null and b/1exa8l2r/wandb/run-20250107_165849-1exa8l2r/run-1exa8l2r.wandb differ diff --git a/2uvisx3r/episode_rosbags/episode_0_2025-01-07-20-34-59.bag b/2uvisx3r/episode_rosbags/episode_0_2025-01-07-20-34-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..89314aa62dbad5ab8d24ab79401e1b8994cd55dc --- /dev/null +++ b/2uvisx3r/episode_rosbags/episode_0_2025-01-07-20-34-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50c4991d37304df66a02fd46b211b3849c2f1152c78c84aa2d7e21858411902f +size 1410130550 diff --git a/2uvisx3r/episode_rosbags/episode_1_2025-01-07-20-35-51.bag b/2uvisx3r/episode_rosbags/episode_1_2025-01-07-20-35-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..9fd5906036af411f7fe81af5e7831878a5e286f4 --- /dev/null +++ b/2uvisx3r/episode_rosbags/episode_1_2025-01-07-20-35-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6553983370638380f1d58c9956c4178a0699a475ca412d805b04f13fbd0d651 +size 1436836895 diff --git a/2uvisx3r/episode_rosbags/episode_3_2025-01-07-20-37-26.bag b/2uvisx3r/episode_rosbags/episode_3_2025-01-07-20-37-26.bag new file mode 100644 index 0000000000000000000000000000000000000000..a5852bf5735fe94051f2b4e519904ae3d10c997b --- /dev/null +++ b/2uvisx3r/episode_rosbags/episode_3_2025-01-07-20-37-26.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca772d06e008ae1be8ce8ccdaa2ac28e5c44be78d9a69daf4d9f0c27fb44346e +size 1412799431 diff --git a/2uvisx3r/episode_rosbags/episode_4_2025-01-07-20-38-14.bag b/2uvisx3r/episode_rosbags/episode_4_2025-01-07-20-38-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..d3bc64627b32e8d87693c3ae075bf33c6097050a --- /dev/null +++ b/2uvisx3r/episode_rosbags/episode_4_2025-01-07-20-38-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b423456ee8fb160c3966276a2774e03c15dea3067dc7b5ecd6f2a2e56de2f719 +size 1443715402 diff --git a/2zm0rh5c/.hydra/config.yaml b/2zm0rh5c/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/2zm0rh5c/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/2zm0rh5c/.hydra/hydra.yaml b/2zm0rh5c/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82a1cdafd5351d543c8bcec95bd40183b96865d4 --- /dev/null +++ b/2zm0rh5c/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/2zm0rh5c/.hydra/overrides.yaml b/2zm0rh5c/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/2zm0rh5c/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/2zm0rh5c/episode_rosbags/aligned_depth_to_color_K.npy b/2zm0rh5c/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/2zm0rh5c/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/2zm0rh5c/episode_rosbags/cam_tf_world.npy b/2zm0rh5c/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/2zm0rh5c/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/2zm0rh5c/episode_rosbags/color_K.npy b/2zm0rh5c/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/2zm0rh5c/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/2zm0rh5c/episode_rosbags/depth_K.npy b/2zm0rh5c/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/2zm0rh5c/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/2zm0rh5c/eval_robot.log b/2zm0rh5c/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..742e4e43baa731f3c68acfee99d12054f596751d --- /dev/null +++ b/2zm0rh5c/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 20:39:28,571][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:39:28,574][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/2zm0rh5c/eval_video/0_eval.mp4 b/2zm0rh5c/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..297d0b2faca68c4816ee16846340701baffd29c7 --- /dev/null +++ b/2zm0rh5c/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85c605b13f98f669d393768cd5cd71b197d1475865f4ce643e59b883b539d36f +size 1066520 diff --git a/2zm0rh5c/eval_video/1_eval.mp4 b/2zm0rh5c/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38b389679dca73cbe35c4cbe0dcc3e52f5eed04c --- /dev/null +++ b/2zm0rh5c/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d9b79e065ade71bc40a9ff8713e4674d2617ef091bf779ecd2d9d0ac4b331bf +size 1141268 diff --git a/2zm0rh5c/eval_video/2_eval.mp4 b/2zm0rh5c/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08d138346547d9a92c5add8667907e6b320511e6 --- /dev/null +++ b/2zm0rh5c/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5363406f4c2180773f400a42cc9880a6c2ddd4b063669bbaf7721762b7f9fce1 +size 1071578 diff --git a/2zm0rh5c/eval_video/3_eval.mp4 b/2zm0rh5c/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc95a6b53338aa2f305ae1a2e596f7a3839a3319 --- /dev/null +++ b/2zm0rh5c/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cb1817dfdfa2909cf482ddf96afca704d995b9dd6d5bf4c86e0008012ac4641 +size 1136484 diff --git a/2zm0rh5c/eval_video/4_eval.mp4 b/2zm0rh5c/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3407fd6622e4b0710a6a82d985a043fe18388edd --- /dev/null +++ b/2zm0rh5c/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8275bd4585ba12c6d3550dcb1f08e6a1a5f05da0c833e4688bd4b33f65f7320b +size 1071859 diff --git a/2zm0rh5c/tb/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 b/2zm0rh5c/tb/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 new file mode 100644 index 0000000000000000000000000000000000000000..1a955ebf3a48884e2f06aab816cb7a05a8e213f2 --- /dev/null +++ b/2zm0rh5c/tb/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39efcb4155c06c358240929f157724319debc9d135e26e5940cab1fa002ae311 +size 6539 diff --git a/2zm0rh5c/wandb/debug-internal.log b/2zm0rh5c/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..46b48c806b56db3d38411152c8926dea39706773 --- /dev/null +++ b/2zm0rh5c/wandb/debug-internal.log @@ -0,0 +1,788 @@ +2025-01-07 20:39:34,369 INFO StreamThr :2229230 [internal.py:wandb_internal():85] W&B internal server running at pid: 2229230, started at: 2025-01-07 20:39:34.369303 +2025-01-07 20:39:34,370 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:39:34,382 INFO WriterThread:2229230 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/run-2zm0rh5c.wandb +2025-01-07 20:39:34,383 DEBUG SenderThread:2229230 [sender.py:send():379] send: header +2025-01-07 20:39:34,394 DEBUG SenderThread:2229230 [sender.py:send():379] send: run +2025-01-07 20:39:34,640 INFO SenderThread:2229230 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files +2025-01-07 20:39:34,640 INFO SenderThread:2229230 [sender.py:_start_run_threads():1188] run started: 2zm0rh5c with start time 1736300374.369338 +2025-01-07 20:39:34,664 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:39:34,664 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:39:34,761 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:39:34,765 DEBUG HandlerThread:2229230 [system_info.py:__init__():26] System info init +2025-01-07 20:39:34,765 DEBUG HandlerThread:2229230 [system_info.py:__init__():41] System info init done +2025-01-07 20:39:34,765 INFO HandlerThread:2229230 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:39:34,766 INFO SystemMonitor:2229230 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:39:34,766 INFO HandlerThread:2229230 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:39:34,766 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:39:34,767 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:39:34,768 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:39:34,768 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:39:34,768 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:39:34,778 DEBUG HandlerThread:2229230 [system_info.py:probe():152] Probing system +2025-01-07 20:39:34,780 DEBUG HandlerThread:2229230 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:39:34,785 DEBUG HandlerThread:2229230 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:39:34,785 DEBUG HandlerThread:2229230 [system_info.py:probe():200] Probing system done +2025-01-07 20:39:34,785 DEBUG HandlerThread:2229230 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:39:34.779023', 'startedAt': '2025-01-08T01:39:34.362109', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3564.010875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4505.107, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.796, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4512.989, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 600.235538482666}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:39:34,785 INFO HandlerThread:2229230 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:39:34,785 INFO HandlerThread:2229230 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:39:34,785 DEBUG HandlerThread:2229230 [system_info.py:_save_code():44] Saving code +2025-01-07 20:39:34,787 DEBUG HandlerThread:2229230 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:39:34,787 DEBUG HandlerThread:2229230 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:39:34,848 DEBUG HandlerThread:2229230 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:39:34,849 INFO HandlerThread:2229230 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:39:34,852 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:39:34,852 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:39:34,852 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:39:34,852 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:39:35,090 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/uhmf2vq5-wandb-metadata.json +2025-01-07 20:39:35,098 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:39:35,098 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:39:35,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:39:35,099 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:39:35,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:35,101 INFO HandlerThread:2229230 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/tb" + save: true +} +control { + relay_id: "f9089ce087b04a57a25ab73a912e7a5a" +} +_info { + stream_id: "2zm0rh5c" +} + +2025-01-07 20:39:35,102 DEBUG HandlerThread:2229230 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:39:35,110 INFO wandb-upload_2:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/qt5zfsb5-diff.patch +2025-01-07 20:39:35,136 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/ceawfdmj-code/FISH/eval_robot.py +2025-01-07 20:39:35,198 DEBUG SenderThread:2229230 [sender.py:send():379] send: telemetry +2025-01-07 20:39:35,198 DEBUG SenderThread:2229230 [sender.py:send():379] send: tbrecord +2025-01-07 20:39:35,198 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:39:35,198 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 with policy live +2025-01-07 20:39:35,386 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/b9o30z0o-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:39:35,641 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code/FISH/eval_robot.py +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-metadata.json +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/diff.patch +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/requirements.txt +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code/FISH +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code +2025-01-07 20:39:36,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:37,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:37,641 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:39:38,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:39,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:40,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:40,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:41,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:42,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:43,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:44,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:45,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:45,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:46,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:47,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:48,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:49,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:50,098 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:39:50,099 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:39:50,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:50,255 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:51,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:52,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:53,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:54,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:55,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:55,719 DEBUG SenderThread:2229230 [sender.py:send():379] send: config +2025-01-07 20:39:55,720 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:56,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:57,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:57,646 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:39:58,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:59,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:59,647 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:00,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:01,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:01,489 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:01,647 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:02,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:03,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:04,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:05,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:40:05,099 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:40:05,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:05,649 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:06,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:07,105 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:07,184 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:07,649 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml +2025-01-07 20:40:07,649 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:08,105 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:09,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:10,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:11,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:11,650 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:12,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:13,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:13,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:14,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:15,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:15,651 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:16,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:17,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:17,652 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:18,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:18,717 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:19,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:20,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:40:20,099 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:40:20,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:21,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:22,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:23,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:24,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:24,192 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:25,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:26,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:27,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:28,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:29,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:30,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:30,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:31,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:32,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:33,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:34,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:34,769 DEBUG SystemMonitor:2229230 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:40:34,769 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:40:35,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:40:35,100 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:40:35,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:35,246 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:36,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:36,830 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:40:36,830 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 with policy now +2025-01-07 20:40:36,831 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:40:36,832 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:40:36,832 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:40:36,832 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:40:37,096 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/j6qqheb3-media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 +2025-01-07 20:40:37,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:37,657 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:37,657 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:40:37,657 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 +2025-01-07 20:40:37,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media +2025-01-07 20:40:37,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos +2025-01-07 20:40:37,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:40:37,975 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/9pdnn7lv-media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 +2025-01-07 20:40:38,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:39,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:39,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:40,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:40,348 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:41,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:41,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:42,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:43,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:44,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:45,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:45,660 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:46,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:46,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:47,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:48,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:49,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:49,661 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:40:49,661 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:49,907 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/hdii67ow-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:40:50,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:40:50,100 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:40:50,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:51,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:51,205 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:51,662 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:52,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:53,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:53,663 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:54,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:55,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:56,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:57,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:57,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:58,113 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:59,113 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:00,113 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:01,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:02,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:02,113 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:03,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:04,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:04,770 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:41:05,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:41:05,100 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:41:05,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:05,667 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:06,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:07,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:07,200 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:08,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:09,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:10,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:11,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:12,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:13,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:13,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:14,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:15,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:16,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:17,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:17,671 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:18,116 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:18,622 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:19,116 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:20,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:41:20,100 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:41:20,116 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:20,833 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:41:20,833 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 with policy now +2025-01-07 20:41:20,835 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:41:20,835 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:41:20,835 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:41:20,837 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:41:21,116 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:21,141 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/1vqam21e-media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 +2025-01-07 20:41:21,672 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:41:21,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:21,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 +2025-01-07 20:41:21,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:41:21,992 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/ao9jus6d-media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 +2025-01-07 20:41:22,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:22,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:23,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:23,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:23,846 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:24,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:25,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:25,674 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:26,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:27,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:28,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:29,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:29,269 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:29,675 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:30,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:31,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:32,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:33,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:34,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:34,770 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:41:34,771 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:35,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:41:35,102 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:41:35,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:36,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:37,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:38,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:39,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:40,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:40,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:41,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:42,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:43,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:44,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:45,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:45,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:45,530 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:41:45,530 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:41:45,531 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:41:45,680 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:41:45,680 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:41:46,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:47,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:47,681 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:48,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:49,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:49,681 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:50,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:41:50,102 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:41:50,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:50,256 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:51,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:51,682 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:52,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:53,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:54,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:55,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:56,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:56,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:57,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:58,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:59,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:00,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:01,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:01,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:02,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:03,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:04,128 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:04,771 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:42:05,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:42:05,102 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:42:05,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:06,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:06,199 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:07,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:08,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:09,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:10,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:11,131 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:12,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:12,128 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:13,128 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:14,129 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:15,128 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:16,132 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:17,040 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:42:17,040 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 with policy now +2025-01-07 20:42:17,042 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:42:17,043 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:42:17,043 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:42:17,044 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:42:17,133 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:17,297 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/8ay4j7p1-media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 +2025-01-07 20:42:17,690 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:42:17,691 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:17,691 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 +2025-01-07 20:42:17,691 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:42:17,997 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/uam2yj2e-media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 +2025-01-07 20:42:18,061 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:18,133 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:19,133 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:19,691 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:20,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:42:20,108 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:42:20,153 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:21,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:21,692 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:22,134 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:23,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:23,291 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:24,137 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:25,139 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:25,693 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:26,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:27,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:28,142 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:29,118 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:29,142 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:29,694 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:42:29,955 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/kikoda2d-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:42:30,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:30,363 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:42:30,363 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:42:30,363 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:42:30,695 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:42:31,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:31,695 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:32,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:33,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:34,142 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:34,593 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:34,771 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:42:35,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:42:35,103 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:42:35,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:35,697 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:36,142 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:37,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:38,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:39,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:40,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:40,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:41,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:42,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:43,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:44,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:45,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:46,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:46,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:47,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:48,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:49,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:50,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:42:50,103 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:42:50,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:51,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:51,200 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:52,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:53,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:54,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:55,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:56,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:57,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:57,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:58,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:59,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:00,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:01,147 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:01,589 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:43:01,589 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:43:01,589 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 with policy now +2025-01-07 20:43:01,590 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:43:01,590 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:43:01,590 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:43:01,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:43:01,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:01,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 +2025-01-07 20:43:01,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:43:01,863 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/c0v8hq5h-media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 +2025-01-07 20:43:02,146 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/t6pmtt4g-media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 +2025-01-07 20:43:02,149 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:02,616 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:03,149 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:03,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:04,153 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:04,772 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:43:05,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:43:05,108 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:43:05,153 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:06,150 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:07,151 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:07,708 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:08,018 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:08,150 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:09,151 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:09,708 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:10,154 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:11,154 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:12,157 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:13,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:13,158 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:14,160 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:15,159 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:16,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:17,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:18,162 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:19,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:19,160 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:20,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:43:20,108 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:43:20,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:21,167 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:22,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:22,429 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:43:22,429 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:43:22,429 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:43:22,713 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:43:22,713 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:43:23,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:23,713 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:24,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:24,385 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:25,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:25,714 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:26,169 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:27,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:27,714 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:28,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:29,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:30,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:30,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:31,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:32,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:33,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:34,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:34,772 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:43:35,105 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:43:35,105 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:43:35,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:35,244 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:36,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:37,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:38,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:39,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:40,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:41,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:41,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:42,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:43,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:44,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:45,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:45,721 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:46,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:46,708 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:47,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:47,722 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:48,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:49,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:50,105 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:43:50,105 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:43:50,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:51,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:52,166 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:52,252 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:53,166 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:54,176 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:54,361 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:43:54,361 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:43:54,361 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 with policy now +2025-01-07 20:43:54,362 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:43:54,362 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:43:54,362 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:43:54,616 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/0zwhdlkj-media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 +2025-01-07 20:43:54,724 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:43:54,724 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 +2025-01-07 20:43:54,724 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:43:55,005 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/1icar29e-media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 +2025-01-07 20:43:55,173 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:55,724 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:56,175 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:57,174 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:57,998 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:58,174 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:59,179 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:59,726 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:00,177 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:01,177 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:01,726 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:02,183 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:03,185 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:03,889 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:04,200 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:04,773 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:44:05,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:44:05,110 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:44:05,204 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:06,193 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:07,202 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:08,199 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:09,070 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:09,196 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:09,483 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:44:09,483 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:09,483 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:09,729 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:09,729 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:09,968 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/noqnef54-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:09,971 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:44:09,972 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:44:10,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:10,203 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:44:10,625 INFO wandb-upload_0:2229230 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpw02_o72k +2025-01-07 20:44:10,637 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:44:10,648 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval +2025-01-07 20:44:11,041 INFO SenderThread:2229230 [sender.py:send_request_log_artifact():1525] logged artifact run-2zm0rh5c-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQxMTgxMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzODQ5Mw==', 'latestArtifact': None}} +2025-01-07 20:44:11,041 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:44:11,041 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:44:11,042 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:44:11,042 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,042 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,043 DEBUG SenderThread:2229230 [sender.py:send():379] send: telemetry +2025-01-07 20:44:11,043 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,046 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,047 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,047 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,047 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,047 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,048 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,048 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,048 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,048 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,049 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,049 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,049 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,049 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,049 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,050 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,050 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,050 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,050 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,051 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,051 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,051 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,051 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:44:11,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:11,231 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/3x2r8qm0-media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json +2025-01-07 20:44:11,250 INFO wandb-upload_2:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/g1z6y3xr-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:44:11,419 INFO wandb-upload_0:2229230 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqgf3nhym +2025-01-07 20:44:11,533 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/16nb8bua-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:44:11,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:11,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:11,788 INFO SenderThread:2229230 [sender.py:send_request_log_artifact():1525] logged artifact run-2zm0rh5c-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQxMTgzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzODQ5NQ==', 'latestArtifact': None}} +2025-01-07 20:44:11,788 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:44:11,788 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json with policy now +2025-01-07 20:44:11,788 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:44:11,788 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,789 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:12,062 INFO wandb-upload_2:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/p4cbp72s-media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json +2025-01-07 20:44:12,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:12,731 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:13,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:13,731 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:14,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:14,789 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:15,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:16,199 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:16,653 DEBUG SenderThread:2229230 [sender.py:send():379] send: exit +2025-01-07 20:44:16,653 INFO SenderThread:2229230 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:send_exit():588] handling runtime: 281 +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:send_exit():594] send defer +2025-01-07 20:44:16,654 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,654 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:44:16,654 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:44:16,655 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,655 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:44:16,655 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:16,655 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:44:16,655 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:44:16,655 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,655 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:44:16,655 INFO HandlerThread:2229230 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:44:16,655 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:44:16,656 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:44:16,656 DEBUG SystemMonitor:2229230 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:44:16,656 DEBUG SystemMonitor:2229230 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:44:16,728 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:44:16,728 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:44:16,728 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:44:16,729 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:16,729 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:44:16,729 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:44:16,729 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:44:16,729 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,729 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:44:16,730 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:16,730 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:44:16,730 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:44:16,730 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,730 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:44:16,732 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:17,733 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:23,488 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:23,488 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:23,488 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:44:23,489 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:23,489 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:44:23,489 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:44:23,489 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:23,489 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:23,490 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:44:23,489 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:44:23,490 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:23,492 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:23,492 DEBUG SenderThread:2229230 [sender.py:send():379] send: summary +2025-01-07 20:44:23,496 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:23,496 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:23,496 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:44:23,496 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:44:23,496 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:23,496 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:44:23,496 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:23,496 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:44:23,500 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:23,615 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:44:23,615 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:23,615 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:44:23,616 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:23,616 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:44:23,654 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:23,735 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:23,735 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml +2025-01-07 20:44:26,837 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:44:26,837 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:26,837 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:26,837 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:44:26,838 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:26,838 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:44:26,838 INFO SenderThread:2229230 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:44:26,840 INFO SenderThread:2229230 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:44:26,842 INFO SenderThread:2229230 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:44:26,853 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:44:26,853 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:26,853 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:44:26,853 DEBUG SenderThread:2229230 [sender.py:send():379] send: artifact +2025-01-07 20:44:27,135 INFO SenderThread:2229230 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:44:27,135 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:27,135 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:44:27,135 INFO SenderThread:2229230 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:44:27,656 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:27,736 INFO SenderThread:2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:27,736 INFO SenderThread:2229230 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files +2025-01-07 20:44:27,737 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml config.yaml +2025-01-07 20:44:27,737 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json wandb-summary.json +2025-01-07 20:44:27,737 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:27,737 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/requirements.txt requirements.txt +2025-01-07 20:44:27,739 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:44:27,739 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log output.log +2025-01-07 20:44:27,739 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/diff.patch diff.patch +2025-01-07 20:44:27,741 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:44:27,741 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 +2025-01-07 20:44:27,743 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 +2025-01-07 20:44:27,743 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 +2025-01-07 20:44:27,743 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 +2025-01-07 20:44:27,743 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 +2025-01-07 20:44:27,746 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json +2025-01-07 20:44:27,746 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:44:27,746 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:44:27,746 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:27,747 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:27,747 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:44:27,747 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:27,747 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:44:27,747 INFO SenderThread:2229230 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:44:27,919 INFO wandb-upload_2:2229230 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:27,965 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml +2025-01-07 20:44:27,967 INFO wandb-upload_4:2229230 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:27,979 INFO wandb-upload_3:2229230 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/requirements.txt +2025-01-07 20:44:28,018 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/hn5fh47k-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:28,218 INFO Thread-11 (_thread_body):2229230 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:44:28,219 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:28,219 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:44:28,219 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:28,219 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:44:28,219 INFO SenderThread:2229230 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:44:28,220 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:44:28,221 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:28,221 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:44:28,221 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:28,221 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:44:28,221 INFO SenderThread:2229230 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:44:28,402 INFO SenderThread:2229230 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:44:28,402 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:44:28,402 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:28,403 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:44:28,403 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:28,403 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:44:28,403 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:44:28,403 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:28,403 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:44:28,403 DEBUG SenderThread:2229230 [sender.py:send():379] send: final +2025-01-07 20:44:28,403 DEBUG SenderThread:2229230 [sender.py:send():379] send: footer +2025-01-07 20:44:28,403 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:28,403 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:44:28,404 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:28,404 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:28,404 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:44:28,404 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:44:28,406 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:28,406 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:44:28,406 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:44:28,407 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:28,454 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:28,454 INFO MainThread:2229230 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:44:28,455 INFO MainThread:2229230 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:44:28,455 INFO MainThread:2229230 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:44:28,455 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:44:28,455 INFO HandlerThread:2229230 [handler.py:finish():882] shutting down handler +2025-01-07 20:44:29,406 INFO WriterThread:2229230 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/run-2zm0rh5c.wandb +2025-01-07 20:44:29,454 INFO SenderThread:2229230 [sender.py:finish():1615] shutting down sender +2025-01-07 20:44:29,455 INFO SenderThread:2229230 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:44:29,455 INFO SenderThread:2229230 [file_pusher.py:join():175] waiting for file pusher diff --git a/2zm0rh5c/wandb/debug.log b/2zm0rh5c/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d3e48cb9fa19ae6749b468827cc2fb272a98378b --- /dev/null +++ b/2zm0rh5c/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Configure stats pid to 2229065 +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/settings +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug.log +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug-internal.log +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c'} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:init():619] starting backend +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:init():623] setting up manager +2025-01-07 20:39:34,368 INFO MainThread:2229065 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:39:34,369 INFO MainThread:2229065 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:39:34,382 INFO MainThread:2229065 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:39:34,392 INFO MainThread:2229065 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:39:34,664 INFO MainThread:2229065 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:39:34,748 INFO MainThread:2229065 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:39:34,749 INFO MainThread:2229065 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:39:35,099 INFO MainThread:2229065 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:39:35,099 INFO MainThread:2229065 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:39:35,099 INFO MainThread:2229065 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:39:35,099 INFO MainThread:2229065 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:39:35,101 INFO MainThread:2229065 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:39:35,101 INFO MainThread:2229065 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/tb, True +2025-01-07 20:39:55,719 INFO MainThread:2229065 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:44:29,456 WARNING MsgRouterThr:2229065 [router.py:message_loop():77] message_loop has been closed diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code/FISH/eval_robot.py b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd85f8bbaf4e9e24125911ec788b8dca6d786df2 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736300374 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/diff.patch b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a994978ad1dd8e2d18bdda54a765e73f2c0b5be --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/diff.patch @@ -0,0 +1,19 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..bc04fba 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -135,12 +135,12 @@ load_checkpoint: ${agent.load_checkpoint} + # RGBD+mask+act history + # wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 +-# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..b44c84c63af25fad6efc8d96ba619adfbd32d3ca --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 60, 0.4166666666666667, 0.2745976746533111, 0.5684240013224129, 0.3003844172161698, 0.5503793209690426, 0.31825467202722046, 0.5206550806719479, 0.3329115137792026, 0.5058161877084066]]} \ No newline at end of file diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..297d0b2faca68c4816ee16846340701baffd29c7 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:85c605b13f98f669d393768cd5cd71b197d1475865f4ce643e59b883b539d36f +size 1066520 diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..38b389679dca73cbe35c4cbe0dcc3e52f5eed04c --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d9b79e065ade71bc40a9ff8713e4674d2617ef091bf779ecd2d9d0ac4b331bf +size 1141268 diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..08d138346547d9a92c5add8667907e6b320511e6 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5363406f4c2180773f400a42cc9880a6c2ddd4b063669bbaf7721762b7f9fce1 +size 1071578 diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fc95a6b53338aa2f305ae1a2e596f7a3839a3319 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cb1817dfdfa2909cf482ddf96afca704d995b9dd6d5bf4c86e0008012ac4641 +size 1136484 diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3407fd6622e4b0710a6a82d985a043fe18388edd --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8275bd4585ba12c6d3550dcb1f08e6a1a5f05da0c833e4688bd4b33f65f7320b +size 1071859 diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a1f0170313c1310d3049a57d8732d9525961ce84 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log @@ -0,0 +1,275 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736300395.719983]: resetting environment +[INFO] [1736300395.722980]: cleared current plan +[INFO] [1736300395.723965]: moving to home +[INFO] [1736300396.726019]: reached home +[INFO] [1736300396.727350]: reset action history +[INFO] [1736300399.435040]: environment reset +Starting episode 0 +[INFO] [1736300399.437350]: resetting environment +[INFO] [1736300399.443358]: cleared current plan +[INFO] [1736300399.444179]: moving to home +[INFO] [1736300400.446490]: reached home +[INFO] [1736300400.447708]: reset action history +[INFO] [1736300403.155263]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736300405.127289]: resetting environment +[INFO] [1736300405.134018]: cleared current plan +[INFO] [1736300405.136275]: moving to home +[INFO] [1736300406.139079]: reached home +[INFO] [1736300406.141540]: reset action history +[INFO] [1736300408.849785]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/episode_rosbags/episode_0_2025-01-07-20-40-08.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2229065.log +started roslaunch server http://158.130.50.37:38157/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2229353] +process[print_text-2]: started with pid [2229354] +process[rosbag_record-3]: started with pid [2229355] +started bagging! +For topic gripper_width: timestamp difference is 101399996 for nearest: 1736300413269372799 - target: 1736300413167972803 at idx 58 +For topic gripper_width: timestamp difference is 106274970 for nearest: 1736300415669362195 - target: 1736300415563087225 at idx 85 +[INFO] [1736300435.856642]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1736300436.611361]: Stored episode 1. +[INFO] [1736300436.611643]: Saving video... +[INFO] [1736300436.818812]: Video saved! +Starting episode 1 +[INFO] [1736300436.833064]: resetting environment +[INFO] [1736300436.844050]: cleared current plan +[INFO] [1736300436.845953]: moving to home +[INFO] [1736300440.348326]: reached home +[INFO] [1736300440.348769]: reset action history +[INFO] [1736300443.056517]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300448.668411]: resetting environment +[INFO] [1736300448.670436]: cleared current plan +[INFO] [1736300448.670612]: moving to home +[INFO] [1736300449.672156]: reached home +[INFO] [1736300449.672637]: reset action history +[INFO] [1736300452.380046]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/episode_rosbags/episode_1_2025-01-07-20-40-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2229065.log +started roslaunch server http://158.130.50.37:43213/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2229511] +process[print_text-5]: started with pid [2229512] +process[rosbag_record-6]: started with pid [2229536] +started bagging! +For topic gripper_width: timestamp difference is 108256105 for nearest: 1736300464169360879 - target: 1736300464061104774 at idx 166 +For topic gripper_width: timestamp difference is 108381842 for nearest: 1736300464369328116 - target: 1736300464260946274 at idx 168 +For topic gripper_width: timestamp difference is 118224600 for nearest: 1736300476569382885 - target: 1736300476451158285 at idx 249 +[INFO] [1736300479.333129]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736300480.093258]: Stored episode 2. +[INFO] [1736300480.093874]: Saving video... +[INFO] [1736300480.810963]: Video saved! +[WARN] [1736300480.661352]: Plan exhausted +[WARN] [1736300480.690228]: Plan exhausted +[WARN] [1736300480.732134]: Plan exhausted +[WARN] [1736300480.769941]: Plan exhausted +[WARN] [1736300480.809345]: Plan exhausted +Starting episode 2 +[INFO] [1736300480.837465]: resetting environment +[INFO] [1736300480.844971]: cleared current plan +[INFO] [1736300480.845506]: moving to home +[INFO] [1736300484.452007]: reached home +[INFO] [1736300484.453608]: reset action history +[INFO] [1736300487.227778]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300504.931342]: resetting environment +[INFO] [1736300504.945260]: cleared current plan +[INFO] [1736300504.945469]: moving to home +[INFO] [1736300505.947506]: reached home +[INFO] [1736300505.948209]: reset action history +[INFO] [1736300508.655209]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/episode_rosbags/episode_2_2025-01-07-20-41-48.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2229065.log +started roslaunch server http://158.130.50.37:46189/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2229665] +process[print_text-8]: started with pid [2229666] +process[rosbag_record-9]: started with pid [2229667] +started bagging! +[INFO] [1736300535.626082]: Storing episode... +[pub_text-7] killing on exit +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[INFO] [1736300536.397002]: Stored episode 3. +[INFO] [1736300536.398215]: Saving video... +[WARN] [1736300536.969752]: Plan exhausted +[WARN] [1736300537.009506]: Plan exhausted +[WARN] [1736300537.049396]: Plan exhausted +[INFO] [1736300537.011432]: Video saved! +Starting episode 3 +[INFO] [1736300537.044050]: resetting environment +[INFO] [1736300537.057580]: cleared current plan +[INFO] [1736300537.060606]: moving to home +[INFO] [1736300540.567219]: reached home +[INFO] [1736300540.570989]: reset action history +[INFO] [1736300543.289842]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300549.451413]: resetting environment +[INFO] [1736300549.463635]: cleared current plan +[INFO] [1736300549.466763]: moving to home +[INFO] [1736300550.469694]: reached home +[INFO] [1736300550.474489]: reset action history +[INFO] [1736300553.189303]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/episode_rosbags/episode_3_2025-01-07-20-42-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2229065.log +started roslaunch server http://158.130.50.37:33823/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2229823] +process[print_text-11]: started with pid [2229824] +process[rosbag_record-12]: started with pid [2229825] +started bagging! +[INFO] [1736300580.326643]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736300581.082082]: Stored episode 4. +[INFO] [1736300581.084784]: Saving video... +[INFO] [1736300581.542529]: Video saved! +Starting episode 4 +[INFO] [1736300581.591639]: resetting environment +[INFO] [1736300581.605922]: cleared current plan +[INFO] [1736300581.612277]: moving to home +[INFO] [1736300585.223524]: reached home +[INFO] [1736300585.228087]: reset action history +[INFO] [1736300588.018207]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736300602.315522]: resetting environment +[INFO] [1736300602.336046]: cleared current plan +[INFO] [1736300602.337337]: moving to home +[INFO] [1736300603.342310]: reached home +[INFO] [1736300603.345920]: reset action history +[INFO] [1736300606.120140]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/episode_rosbags/episode_4_2025-01-07-20-43-26.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2229065.log +started roslaunch server http://158.130.50.37:45599/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2229936] +process[print_text-14]: started with pid [2229937] +process[rosbag_record-15]: started with pid [2229961] +started bagging! +For topic gripper_width: timestamp difference is 101028048 for nearest: 1736300624902705752 - target: 1736300624801677704 at idx 245 +For topic gripper_width: timestamp difference is 104700722 for nearest: 1736300626702682890 - target: 1736300626597982168 at idx 249 +[INFO] [1736300633.109365]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1736300633.884272]: Stored episode 5. +[INFO] [1736300633.892247]: Saving video... +[INFO] [1736300634.313719]: Video saved! +[INFO] [1736300634.352291]: resetting environment +[INFO] [1736300634.376673]: cleared current plan +[INFO] [1736300634.379627]: moving to home +[INFO] [1736300637.997293]: reached home +[INFO] [1736300638.001201]: reset action history +[INFO] [1736300640.843435]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/requirements.txt b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-metadata.json b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6687835d734f6e6794b68b96875043a69a35f920 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:39:34.779023", + "startedAt": "2025-01-08T01:39:34.362109", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3564.010875, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4505.107, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.796, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4512.989, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 600.235538482666 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..57c47b4a1a701d1720481f691b7fb048abce966c --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "85c605b13f98f669d393768cd5cd71b197d1475865f4ce643e59b883b539d36f", "size": 1066520, "path": "media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4"}, "global_step": 665, "_timestamp": 1736300650.5480618, "_runtime": 276.17872381210327, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "9d9b79e065ade71bc40a9ff8713e4674d2617ef091bf779ecd2d9d0ac4b331bf", "size": 1141268, "path": "media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4"}, "eval/num_success": 0.0, "episode": 60.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "5363406f4c2180773f400a42cc9880a6c2ddd4b063669bbaf7721762b7f9fce1", "size": 1071578, "path": "media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "8cb1817dfdfa2909cf482ddf96afca704d995b9dd6d5bf4c86e0008012ac4641", "size": 1136484, "path": "media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "8275bd4585ba12c6d3550dcb1f08e6a1a5f05da0c833e4688bd4b33f65f7320b", "size": 1071859, "path": "media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://1qhukftn14vhdqm2qdgbzj42jpj6jzmakdocoj5blnjzgqbn9fn1z33nvnhzo6zexwyw8rm6ztr7hqh64w8uj1sj1rkp4aee9zbtp4dxy6m3ikbvszaunu4hkdxgr5wu/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://uflz2id71ciu5bbg6kpb2v4erpgjc32ngdujzx9vjea4dx6ao298i3m91iwp2ca5qu23y7n29nttv96gq5xk7fqsmqk3i7ywqlnve7f8f20hcgqhbli4uq51htjqlo22:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 60, "total_success_rate": 0.4166666666666667, "total_success_rate_lb_0.01": 0.2745976746533111, "total_success_rate_ub_0.01": 0.5684240013224129, "total_success_rate_lb_0.025": 0.3003844172161698, "total_success_rate_ub_0.025": 0.5503793209690426, "total_success_rate_lb_0.05": 0.31825467202722046, "total_success_rate_ub_0.05": 0.5206550806719479, "total_success_rate_lb_0.1": 0.3329115137792026, "total_success_rate_ub_0.1": 0.5058161877084066, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "864ed14701316e15761f658e7f4f1632ddaad90bb3a3498dc31f50b497f6c9e1", "size": 521, "artifact_path": "wandb-client-artifact://ppkbjettrgn4sjpvkta2z8fys7z465rs77x1iqtmjpo4k30vsz2iiz006kojbxkvxlwgernsdfc8oxwsc65xjrtuz7ggmkn129jsxj97pmmujzqmv7xntj5oqnog9h20/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vcl03qvhozrycfv9rd1t84e8bow332erokiknwu73pab3k5hnwtbq2lycaik6eghdqwwdnsaedv7zs69gpl10txrty0p95z6iyku7j39hrndeecwp3en8w3elhy6ymzw:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 281}, "eval/total_success_rate": 0.4166666567325592, "eval/total_success_rate_lb_0.01": 0.2745976746082306, "eval/total_success_rate_ub_0.01": 0.5684239864349365, "eval/total_success_rate_lb_0.025": 0.30038443207740784, "eval/total_success_rate_ub_0.025": 0.5503793358802795, "eval/total_success_rate_lb_0.05": 0.318254679441452, "eval/total_success_rate_ub_0.05": 0.52065509557724, "eval/total_success_rate_lb_0.1": 0.33291152119636536, "eval/total_success_rate_ub_0.1": 0.5058161616325378} \ No newline at end of file diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug-internal.log b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..46b48c806b56db3d38411152c8926dea39706773 --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug-internal.log @@ -0,0 +1,788 @@ +2025-01-07 20:39:34,369 INFO StreamThr :2229230 [internal.py:wandb_internal():85] W&B internal server running at pid: 2229230, started at: 2025-01-07 20:39:34.369303 +2025-01-07 20:39:34,370 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:39:34,382 INFO WriterThread:2229230 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/run-2zm0rh5c.wandb +2025-01-07 20:39:34,383 DEBUG SenderThread:2229230 [sender.py:send():379] send: header +2025-01-07 20:39:34,394 DEBUG SenderThread:2229230 [sender.py:send():379] send: run +2025-01-07 20:39:34,640 INFO SenderThread:2229230 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files +2025-01-07 20:39:34,640 INFO SenderThread:2229230 [sender.py:_start_run_threads():1188] run started: 2zm0rh5c with start time 1736300374.369338 +2025-01-07 20:39:34,664 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:39:34,664 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:39:34,761 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:39:34,765 DEBUG HandlerThread:2229230 [system_info.py:__init__():26] System info init +2025-01-07 20:39:34,765 DEBUG HandlerThread:2229230 [system_info.py:__init__():41] System info init done +2025-01-07 20:39:34,765 INFO HandlerThread:2229230 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:39:34,766 INFO SystemMonitor:2229230 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:39:34,766 INFO HandlerThread:2229230 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:39:34,766 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:39:34,767 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:39:34,768 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:39:34,768 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:39:34,768 INFO SystemMonitor:2229230 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:39:34,778 DEBUG HandlerThread:2229230 [system_info.py:probe():152] Probing system +2025-01-07 20:39:34,780 DEBUG HandlerThread:2229230 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:39:34,785 DEBUG HandlerThread:2229230 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:39:34,785 DEBUG HandlerThread:2229230 [system_info.py:probe():200] Probing system done +2025-01-07 20:39:34,785 DEBUG HandlerThread:2229230 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:39:34.779023', 'startedAt': '2025-01-08T01:39:34.362109', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3564.010875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4505.107, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.796, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4512.989, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 600.235538482666}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:39:34,785 INFO HandlerThread:2229230 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:39:34,785 INFO HandlerThread:2229230 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:39:34,785 DEBUG HandlerThread:2229230 [system_info.py:_save_code():44] Saving code +2025-01-07 20:39:34,787 DEBUG HandlerThread:2229230 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:39:34,787 DEBUG HandlerThread:2229230 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:39:34,848 DEBUG HandlerThread:2229230 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:39:34,849 INFO HandlerThread:2229230 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:39:34,852 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:39:34,852 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:39:34,852 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:39:34,852 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:39:35,090 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/uhmf2vq5-wandb-metadata.json +2025-01-07 20:39:35,098 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:39:35,098 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:39:35,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:39:35,099 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:39:35,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:35,101 INFO HandlerThread:2229230 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/tb" + save: true +} +control { + relay_id: "f9089ce087b04a57a25ab73a912e7a5a" +} +_info { + stream_id: "2zm0rh5c" +} + +2025-01-07 20:39:35,102 DEBUG HandlerThread:2229230 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:39:35,110 INFO wandb-upload_2:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/qt5zfsb5-diff.patch +2025-01-07 20:39:35,136 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/ceawfdmj-code/FISH/eval_robot.py +2025-01-07 20:39:35,198 DEBUG SenderThread:2229230 [sender.py:send():379] send: telemetry +2025-01-07 20:39:35,198 DEBUG SenderThread:2229230 [sender.py:send():379] send: tbrecord +2025-01-07 20:39:35,198 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:39:35,198 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 with policy live +2025-01-07 20:39:35,386 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/b9o30z0o-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:39:35,641 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code/FISH/eval_robot.py +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-metadata.json +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/diff.patch +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/requirements.txt +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code/FISH +2025-01-07 20:39:35,642 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code +2025-01-07 20:39:36,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:37,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:37,641 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:39:38,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:39,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:40,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:40,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:41,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:42,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:43,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:44,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:45,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:45,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:46,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:47,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:48,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:49,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:50,098 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:39:50,099 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:39:50,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:50,255 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:51,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:52,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:53,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:54,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:55,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:55,719 DEBUG SenderThread:2229230 [sender.py:send():379] send: config +2025-01-07 20:39:55,720 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:39:56,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:57,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:57,646 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:39:58,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:59,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:39:59,647 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:00,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:01,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:01,489 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:01,647 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:02,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:03,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:04,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:05,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:40:05,099 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:40:05,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:05,649 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:06,104 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:07,105 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:07,184 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:07,649 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml +2025-01-07 20:40:07,649 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:08,105 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:09,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:10,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:11,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:11,650 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:12,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:13,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:13,106 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:14,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:15,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:15,651 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:16,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:17,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:17,652 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:18,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:18,717 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:19,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:20,099 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:40:20,099 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:40:20,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:21,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:22,107 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:23,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:24,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:24,192 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:25,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:26,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:27,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:28,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:29,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:30,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:30,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:31,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:32,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:33,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:34,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:34,769 DEBUG SystemMonitor:2229230 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:40:34,769 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:40:35,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:40:35,100 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:40:35,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:35,246 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:36,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:36,830 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:40:36,830 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 with policy now +2025-01-07 20:40:36,831 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:40:36,832 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:40:36,832 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:40:36,832 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:40:37,096 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/j6qqheb3-media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 +2025-01-07 20:40:37,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:37,657 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:37,657 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:40:37,657 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 +2025-01-07 20:40:37,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media +2025-01-07 20:40:37,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos +2025-01-07 20:40:37,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:40:37,975 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/9pdnn7lv-media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 +2025-01-07 20:40:38,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:39,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:39,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:40,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:40,348 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:41,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:41,658 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:42,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:43,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:44,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:45,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:45,660 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:46,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:46,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:47,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:48,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:49,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:49,661 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:40:49,661 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:49,907 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/hdii67ow-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:40:50,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:40:50,100 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:40:50,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:51,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:51,205 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:51,662 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:52,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:53,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:53,663 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:40:54,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:55,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:56,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:57,101 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:40:57,112 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:58,113 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:40:59,113 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:00,113 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:01,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:02,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:02,113 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:03,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:04,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:04,770 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:41:05,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:41:05,100 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:41:05,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:05,667 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:06,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:07,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:07,200 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:08,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:09,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:10,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:11,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:12,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:13,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:13,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:14,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:15,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:16,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:17,115 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:17,671 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:18,116 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:18,622 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:19,116 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:20,100 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:41:20,100 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:41:20,116 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:20,833 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:41:20,833 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 with policy now +2025-01-07 20:41:20,835 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:41:20,835 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:41:20,835 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:41:20,837 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:41:21,116 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:21,141 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/1vqam21e-media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 +2025-01-07 20:41:21,672 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:41:21,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:21,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 +2025-01-07 20:41:21,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:41:21,992 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/ao9jus6d-media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 +2025-01-07 20:41:22,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:22,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:23,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:23,673 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:23,846 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:24,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:25,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:25,674 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:26,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:27,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:28,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:29,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:29,269 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:29,675 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:30,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:31,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:32,119 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:33,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:34,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:34,770 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:41:34,771 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:35,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:41:35,102 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:41:35,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:36,120 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:37,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:38,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:39,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:40,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:40,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:41,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:42,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:43,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:44,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:45,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:45,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:45,530 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:41:45,530 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:41:45,531 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:41:45,680 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:41:45,680 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:41:46,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:47,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:47,681 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:48,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:49,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:49,681 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:50,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:41:50,102 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:41:50,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:50,256 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:51,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:51,682 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:41:52,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:53,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:54,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:55,123 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:56,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:41:56,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:57,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:58,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:41:59,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:00,124 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:01,110 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:01,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:02,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:03,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:04,128 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:04,771 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:42:05,102 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:42:05,102 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:42:05,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:06,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:06,199 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:07,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:08,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:09,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:10,125 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:11,131 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:12,111 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:12,128 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:13,128 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:14,129 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:15,128 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:16,132 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:17,040 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:42:17,040 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 with policy now +2025-01-07 20:42:17,042 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:42:17,043 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:42:17,043 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:42:17,044 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:42:17,133 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:17,297 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/8ay4j7p1-media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 +2025-01-07 20:42:17,690 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:42:17,691 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:17,691 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 +2025-01-07 20:42:17,691 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:42:17,997 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/uam2yj2e-media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 +2025-01-07 20:42:18,061 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:18,133 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:19,133 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:19,691 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:20,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:42:20,108 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:42:20,153 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:21,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:21,692 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:22,134 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:23,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:23,291 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:24,137 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:25,139 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:25,693 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:26,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:27,141 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:28,142 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:29,118 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:29,142 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:29,694 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:42:29,955 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/kikoda2d-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:42:30,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:30,363 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:42:30,363 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:42:30,363 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:42:30,695 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:42:31,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:31,695 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:32,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:33,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:34,142 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:34,593 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:34,771 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:42:35,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:42:35,103 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:42:35,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:35,697 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:42:36,142 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:37,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:38,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:39,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:40,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:40,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:41,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:42,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:43,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:44,143 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:45,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:46,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:46,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:47,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:48,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:49,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:50,103 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:42:50,103 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:42:50,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:51,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:51,200 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:52,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:53,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:54,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:55,144 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:56,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:57,114 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:42:57,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:58,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:42:59,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:00,145 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:01,147 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:01,589 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:43:01,589 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:43:01,589 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 with policy now +2025-01-07 20:43:01,590 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:43:01,590 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:43:01,590 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:43:01,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:43:01,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:01,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 +2025-01-07 20:43:01,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:43:01,863 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/c0v8hq5h-media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 +2025-01-07 20:43:02,146 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/t6pmtt4g-media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 +2025-01-07 20:43:02,149 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:02,616 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:03,149 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:03,706 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:04,153 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:04,772 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:43:05,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:43:05,108 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:43:05,153 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:06,150 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:07,151 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:07,708 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:08,018 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:08,150 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:09,151 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:09,708 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:10,154 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:11,154 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:12,157 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:13,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:13,158 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:14,160 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:15,159 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:16,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:17,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:18,162 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:19,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:19,160 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:20,108 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:43:20,108 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:43:20,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:21,167 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:22,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:22,429 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:43:22,429 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:43:22,429 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:43:22,713 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:43:22,713 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:43:23,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:23,713 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:24,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:24,385 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:25,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:25,714 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:26,169 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:27,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:27,714 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:28,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:29,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:30,121 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:30,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:31,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:32,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:33,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:34,163 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:34,772 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:43:35,105 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:43:35,105 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:43:35,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:35,244 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:36,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:37,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:38,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:39,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:40,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:41,122 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:41,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:42,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:43,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:44,164 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:45,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:45,721 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:46,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:46,708 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:47,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:47,722 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:48,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:49,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:50,105 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:43:50,105 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:43:50,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:51,165 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:52,166 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:52,252 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:53,166 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:54,176 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:54,361 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:43:54,361 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:43:54,361 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 with policy now +2025-01-07 20:43:54,362 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:43:54,362 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:43:54,362 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:43:54,616 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/0zwhdlkj-media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 +2025-01-07 20:43:54,724 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:43:54,724 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 +2025-01-07 20:43:54,724 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval +2025-01-07 20:43:55,005 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/1icar29e-media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 +2025-01-07 20:43:55,173 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:55,724 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:43:56,175 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:57,174 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:57,998 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:43:58,174 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:59,179 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:43:59,726 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:00,177 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:01,177 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:01,726 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:02,183 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:03,185 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:03,889 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:04,200 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:04,773 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:44:05,109 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:44:05,110 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:44:05,204 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:06,193 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:07,202 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:08,199 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:09,070 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:09,196 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:09,483 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:44:09,483 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:09,483 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:09,729 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:09,729 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:09,968 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/noqnef54-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:09,971 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:44:09,972 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:44:10,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:10,203 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:44:10,625 INFO wandb-upload_0:2229230 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpw02_o72k +2025-01-07 20:44:10,637 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:44:10,648 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table +2025-01-07 20:44:10,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval +2025-01-07 20:44:11,041 INFO SenderThread:2229230 [sender.py:send_request_log_artifact():1525] logged artifact run-2zm0rh5c-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQxMTgxMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzODQ5Mw==', 'latestArtifact': None}} +2025-01-07 20:44:11,041 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:44:11,041 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-07 20:44:11,042 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:44:11,042 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,042 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,043 DEBUG SenderThread:2229230 [sender.py:send():379] send: telemetry +2025-01-07 20:44:11,043 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,046 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,047 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,047 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,047 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,047 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,048 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,048 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,048 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,048 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,049 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,049 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,049 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,049 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,049 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,050 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,050 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,050 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,050 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,051 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,051 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,051 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:11,051 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:44:11,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:11,231 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/3x2r8qm0-media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json +2025-01-07 20:44:11,250 INFO wandb-upload_2:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/g1z6y3xr-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:44:11,419 INFO wandb-upload_0:2229230 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqgf3nhym +2025-01-07 20:44:11,533 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/16nb8bua-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:44:11,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:11,730 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:11,788 INFO SenderThread:2229230 [sender.py:send_request_log_artifact():1525] logged artifact run-2zm0rh5c-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjQxMTgzMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzODQ5NQ==', 'latestArtifact': None}} +2025-01-07 20:44:11,788 DEBUG SenderThread:2229230 [sender.py:send():379] send: files +2025-01-07 20:44:11,788 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json with policy now +2025-01-07 20:44:11,788 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:44:11,788 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:11,789 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:12,062 INFO wandb-upload_2:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/p4cbp72s-media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json +2025-01-07 20:44:12,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:12,731 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:13,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:13,731 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:14,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:14,789 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:15,198 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:16,199 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:16,653 DEBUG SenderThread:2229230 [sender.py:send():379] send: exit +2025-01-07 20:44:16,653 INFO SenderThread:2229230 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:send_exit():588] handling runtime: 281 +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:send_exit():594] send defer +2025-01-07 20:44:16,654 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,654 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:44:16,654 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:44:16,654 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:44:16,655 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,655 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:44:16,655 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:16,655 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:44:16,655 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:44:16,655 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,655 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:44:16,655 INFO HandlerThread:2229230 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:44:16,655 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:44:16,656 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:44:16,656 DEBUG SystemMonitor:2229230 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:44:16,656 DEBUG SystemMonitor:2229230 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:44:16,728 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:44:16,728 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:44:16,728 INFO HandlerThread:2229230 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:44:16,729 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:16,729 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:44:16,729 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:44:16,729 DEBUG SenderThread:2229230 [sender.py:send():379] send: stats +2025-01-07 20:44:16,729 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,729 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:44:16,730 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:16,730 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:44:16,730 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:44:16,730 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:16,730 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:44:16,732 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:17,733 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:23,488 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:23,488 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:23,488 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:44:23,489 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:23,489 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:44:23,489 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:44:23,489 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:23,489 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:23,490 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:44:23,489 DEBUG SenderThread:2229230 [sender.py:send():379] send: history +2025-01-07 20:44:23,490 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:44:23,492 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:23,492 DEBUG SenderThread:2229230 [sender.py:send():379] send: summary +2025-01-07 20:44:23,496 INFO SenderThread:2229230 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:44:23,496 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:23,496 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:44:23,496 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:44:23,496 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:23,496 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:44:23,496 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:23,496 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:44:23,500 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:44:23,615 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:44:23,615 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:23,615 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:44:23,616 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:23,616 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:44:23,654 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:23,735 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:23,735 INFO Thread-12 :2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml +2025-01-07 20:44:26,837 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:44:26,837 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:26,837 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:26,837 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:44:26,838 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:26,838 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:44:26,838 INFO SenderThread:2229230 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:44:26,840 INFO SenderThread:2229230 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:44:26,842 INFO SenderThread:2229230 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:44:26,853 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:44:26,853 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:26,853 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:44:26,853 DEBUG SenderThread:2229230 [sender.py:send():379] send: artifact +2025-01-07 20:44:27,135 INFO SenderThread:2229230 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:44:27,135 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:27,135 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:44:27,135 INFO SenderThread:2229230 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:44:27,656 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:27,736 INFO SenderThread:2229230 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:27,736 INFO SenderThread:2229230 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files +2025-01-07 20:44:27,737 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml config.yaml +2025-01-07 20:44:27,737 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json wandb-summary.json +2025-01-07 20:44:27,737 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:27,737 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/requirements.txt requirements.txt +2025-01-07 20:44:27,739 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:44:27,739 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log output.log +2025-01-07 20:44:27,739 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/diff.patch diff.patch +2025-01-07 20:44:27,741 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:44:27,741 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 media/videos/eval/3_eval_3_8cb1817dfdfa2909cf48.mp4 +2025-01-07 20:44:27,743 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 media/videos/eval/1_eval_1_9d9b79e065ade71bc40a.mp4 +2025-01-07 20:44:27,743 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 media/videos/eval/4_eval_4_8275bd4585ba12c6d355.mp4 +2025-01-07 20:44:27,743 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 media/videos/eval/0_eval_0_85c605b13f98f669d393.mp4 +2025-01-07 20:44:27,743 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 media/videos/eval/2_eval_2_5363406f4c2180773f40.mp4 +2025-01-07 20:44:27,746 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json media/table/eval/total_success_rate_ci_6_864ed14701316e15761f.table.json +2025-01-07 20:44:27,746 INFO SenderThread:2229230 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-07 20:44:27,746 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:44:27,746 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:27,747 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:27,747 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:44:27,747 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:27,747 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:44:27,747 INFO SenderThread:2229230 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:44:27,919 INFO wandb-upload_2:2229230 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/wandb-summary.json +2025-01-07 20:44:27,965 INFO wandb-upload_1:2229230 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/config.yaml +2025-01-07 20:44:27,967 INFO wandb-upload_4:2229230 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/output.log +2025-01-07 20:44:27,979 INFO wandb-upload_3:2229230 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/files/requirements.txt +2025-01-07 20:44:28,018 INFO wandb-upload_0:2229230 [upload_job.py:push():130] Uploaded file /tmp/tmpfeafvfpowandb/hn5fh47k-events.out.tfevents.1736300375.leonmkim-ROG-Strix-G15CS-G15CS.2229065.0 +2025-01-07 20:44:28,218 INFO Thread-11 (_thread_body):2229230 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:44:28,219 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:28,219 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:44:28,219 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:28,219 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:44:28,219 INFO SenderThread:2229230 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:44:28,220 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:44:28,221 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:28,221 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:44:28,221 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:28,221 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:44:28,221 INFO SenderThread:2229230 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:44:28,402 INFO SenderThread:2229230 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:44:28,402 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:44:28,402 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:28,403 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:44:28,403 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:28,403 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:44:28,403 INFO SenderThread:2229230 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:44:28,403 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:44:28,403 INFO HandlerThread:2229230 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:44:28,403 DEBUG SenderThread:2229230 [sender.py:send():379] send: final +2025-01-07 20:44:28,403 DEBUG SenderThread:2229230 [sender.py:send():379] send: footer +2025-01-07 20:44:28,403 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: defer +2025-01-07 20:44:28,403 INFO SenderThread:2229230 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:44:28,404 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:28,404 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:28,404 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:44:28,404 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:44:28,406 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:44:28,406 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:44:28,406 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:44:28,407 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:44:28,454 DEBUG SenderThread:2229230 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:44:28,454 INFO MainThread:2229230 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:44:28,455 INFO MainThread:2229230 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:44:28,455 INFO MainThread:2229230 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:44:28,455 DEBUG HandlerThread:2229230 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:44:28,455 INFO HandlerThread:2229230 [handler.py:finish():882] shutting down handler +2025-01-07 20:44:29,406 INFO WriterThread:2229230 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/run-2zm0rh5c.wandb +2025-01-07 20:44:29,454 INFO SenderThread:2229230 [sender.py:finish():1615] shutting down sender +2025-01-07 20:44:29,455 INFO SenderThread:2229230 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:44:29,455 INFO SenderThread:2229230 [file_pusher.py:join():175] waiting for file pusher diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug.log b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d3e48cb9fa19ae6749b468827cc2fb272a98378b --- /dev/null +++ b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Configure stats pid to 2229065 +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/settings +2025-01-07 20:39:34,363 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug.log +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/logs/debug-internal.log +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c'} +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:init():619] starting backend +2025-01-07 20:39:34,364 INFO MainThread:2229065 [wandb_init.py:init():623] setting up manager +2025-01-07 20:39:34,368 INFO MainThread:2229065 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:39:34,369 INFO MainThread:2229065 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:39:34,382 INFO MainThread:2229065 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:39:34,392 INFO MainThread:2229065 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:39:34,664 INFO MainThread:2229065 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:39:34,748 INFO MainThread:2229065 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:39:34,749 INFO MainThread:2229065 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:39:35,099 INFO MainThread:2229065 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:39:35,099 INFO MainThread:2229065 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:39:35,099 INFO MainThread:2229065 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:39:35,099 INFO MainThread:2229065 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:39:35,101 INFO MainThread:2229065 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:39:35,101 INFO MainThread:2229065 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/2zm0rh5c/tb, True +2025-01-07 20:39:55,719 INFO MainThread:2229065 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:44:29,456 WARNING MsgRouterThr:2229065 [router.py:message_loop():77] message_loop has been closed diff --git a/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/run-2zm0rh5c.wandb b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/run-2zm0rh5c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..efc8e9510c0a6fdeaef91dad9379e5b60a83337e Binary files /dev/null and b/2zm0rh5c/wandb/run-20250107_203934-2zm0rh5c/run-2zm0rh5c.wandb differ diff --git a/917c47lf/.hydra/config.yaml b/917c47lf/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/917c47lf/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/917c47lf/.hydra/hydra.yaml b/917c47lf/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7476d50840655cbafbdf2ac345dc1ea0ba3d2e78 --- /dev/null +++ b/917c47lf/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/917c47lf/.hydra/overrides.yaml b/917c47lf/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/917c47lf/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/917c47lf/episode_rosbags/episode_1_2025-01-06-22-19-14.bag b/917c47lf/episode_rosbags/episode_1_2025-01-06-22-19-14.bag new file mode 100644 index 0000000000000000000000000000000000000000..a7e9e8abe04c5dd7f9a5e030f05cc720e1c810ae --- /dev/null +++ b/917c47lf/episode_rosbags/episode_1_2025-01-06-22-19-14.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a96933c68c331e515db09abe1022393da7618d108ace8a59a2b253a1a5d49161 +size 1412612848 diff --git a/917c47lf/episode_rosbags/episode_2_2025-01-06-22-19-54.bag b/917c47lf/episode_rosbags/episode_2_2025-01-06-22-19-54.bag new file mode 100644 index 0000000000000000000000000000000000000000..58ebd28a220184e02bd430f511bfddb00ecc70ef --- /dev/null +++ b/917c47lf/episode_rosbags/episode_2_2025-01-06-22-19-54.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13dbbe356c798c5b8ed6bb76c313cc01a1e8fb40ebad3c118c088b9b62382324 +size 1436908347 diff --git a/917c47lf/episode_rosbags/episode_3_2025-01-06-22-20-57.bag b/917c47lf/episode_rosbags/episode_3_2025-01-06-22-20-57.bag new file mode 100644 index 0000000000000000000000000000000000000000..c470889162c0219abacbee3ef8f51b0bf641cdd3 --- /dev/null +++ b/917c47lf/episode_rosbags/episode_3_2025-01-06-22-20-57.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bd6803e1e2c75f60d721262f19f172e152625b3a4a1fac9b1b24722783b595cd +size 1408636211 diff --git a/917c47lf/episode_rosbags/episode_4_2025-01-06-22-21-45.bag b/917c47lf/episode_rosbags/episode_4_2025-01-06-22-21-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..ffc53b5291d261ff054312a9c4baefe5565ae756 --- /dev/null +++ b/917c47lf/episode_rosbags/episode_4_2025-01-06-22-21-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fecae5c05aba9ffd6a42c5a348d9ad0ddfdc00a37946dff090adaabf9dcbc34f +size 1456950397 diff --git a/917c47lf/eval_video/0_eval.mp4 b/917c47lf/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa9e7ff22c305e05ebc26305bf5edd77cdaaaf5e --- /dev/null +++ b/917c47lf/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51de33a49d1b3553fbba41c97b8700d45a16e34447711639790a2881dcf2c6a3 +size 1188873 diff --git a/917c47lf/eval_video/1_eval.mp4 b/917c47lf/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ee875ae28d638c9c9ced4cbbc1f2884175c808e --- /dev/null +++ b/917c47lf/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a016504166fce504de7265eb372b780204054bfb5768857df26ed6e1c0d442c +size 1189054 diff --git a/917c47lf/eval_video/2_eval.mp4 b/917c47lf/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e61b70d839ed7328cfd7e21f27aa26a6af3e7676 --- /dev/null +++ b/917c47lf/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e8e215f602a2441e87a870098065953bee6249823d409431e7d40031381d4b3 +size 1304735 diff --git a/917c47lf/eval_video/3_eval.mp4 b/917c47lf/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a15961554b4169c258b7d4a04ef754b952009a6 --- /dev/null +++ b/917c47lf/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cfc3fdd64623711fec5c4b51ef4a0c2497c8c628ded5e0279dc9eae3cff9f66 +size 1144448 diff --git a/917c47lf/eval_video/4_eval.mp4 b/917c47lf/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..20bf18421d74da7bec05aa019b1cb1e635031a91 --- /dev/null +++ b/917c47lf/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3e93442594f3e5f3d1e87d8fea5936129990d994d930dde5165fb385ac3e0a0 +size 1238915 diff --git a/917c47lf/tb/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 b/917c47lf/tb/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 new file mode 100644 index 0000000000000000000000000000000000000000..d61b10f3d79108077dd0825813b8026abe447608 --- /dev/null +++ b/917c47lf/tb/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:237063c435c3593384d5f499accdf3ac478e11e8838635ac5a6e5401eb330405 +size 6539 diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/code/FISH/eval_robot.py b/917c47lf/wandb/run-20250106_221810-917c47lf/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json new file mode 100644 index 0000000000000000000000000000000000000000..30d70111fb6d2524d04966b9af5719ddb91b0b18 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.003681928018651432, 0.7286170720596843, 0.0051474646723538384, 0.645829737157266, 0.025117173885908266, 0.5745371579974826, 0.10005546369206852, 0.4711491614637855]]} \ No newline at end of file diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json new file mode 100644 index 0000000000000000000000000000000000000000..d4f4661278a7f266f24e4e2882c988041e975070 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[11, 25, 0.44, 0.22379118209640997, 0.6720014512194907, 0.2644356638664259, 0.649185761779512, 0.29046089951414444, 0.5987451448839108, 0.3072170541075988, 0.5827415138317564]]} \ No newline at end of file diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..aa9e7ff22c305e05ebc26305bf5edd77cdaaaf5e --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:51de33a49d1b3553fbba41c97b8700d45a16e34447711639790a2881dcf2c6a3 +size 1188873 diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0ee875ae28d638c9c9ced4cbbc1f2884175c808e --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a016504166fce504de7265eb372b780204054bfb5768857df26ed6e1c0d442c +size 1189054 diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e61b70d839ed7328cfd7e21f27aa26a6af3e7676 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e8e215f602a2441e87a870098065953bee6249823d409431e7d40031381d4b3 +size 1304735 diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a15961554b4169c258b7d4a04ef754b952009a6 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cfc3fdd64623711fec5c4b51ef4a0c2497c8c628ded5e0279dc9eae3cff9f66 +size 1144448 diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..20bf18421d74da7bec05aa019b1cb1e635031a91 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3e93442594f3e5f3d1e87d8fea5936129990d994d930dde5165fb385ac3e0a0 +size 1238915 diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug-internal.log b/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b322029fa0a3a4f25a70bfa63b54b4ba8a463004 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug-internal.log @@ -0,0 +1,767 @@ +2025-01-06 22:18:10,159 INFO StreamThr :2123564 [internal.py:wandb_internal():85] W&B internal server running at pid: 2123564, started at: 2025-01-06 22:18:10.159088 +2025-01-06 22:18:10,160 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:18:10,172 INFO WriterThread:2123564 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/run-917c47lf.wandb +2025-01-06 22:18:10,172 DEBUG SenderThread:2123564 [sender.py:send():379] send: header +2025-01-06 22:18:10,181 DEBUG SenderThread:2123564 [sender.py:send():379] send: run +2025-01-06 22:18:10,412 INFO SenderThread:2123564 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files +2025-01-06 22:18:10,413 INFO SenderThread:2123564 [sender.py:_start_run_threads():1188] run started: 917c47lf with start time 1736219890.1595 +2025-01-06 22:18:10,429 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:18:10,429 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:18:10,499 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:18:10,504 DEBUG HandlerThread:2123564 [system_info.py:__init__():26] System info init +2025-01-06 22:18:10,504 DEBUG HandlerThread:2123564 [system_info.py:__init__():41] System info init done +2025-01-06 22:18:10,504 INFO HandlerThread:2123564 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:18:10,504 INFO SystemMonitor:2123564 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:18:10,504 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:18:10,504 INFO HandlerThread:2123564 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:18:10,505 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:18:10,506 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:18:10,507 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:18:10,507 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:18:10,518 DEBUG HandlerThread:2123564 [system_info.py:probe():152] Probing system +2025-01-06 22:18:10,520 DEBUG HandlerThread:2123564 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:18:10,524 DEBUG HandlerThread:2123564 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:18:10,524 DEBUG HandlerThread:2123564 [system_info.py:probe():200] Probing system done +2025-01-06 22:18:10,524 DEBUG HandlerThread:2123564 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:18:10.518603', 'startedAt': '2025-01-07T03:18:10.150271', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.4985, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 4500.056, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 564.8245735168457}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:18:10,524 INFO HandlerThread:2123564 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:18:10,524 INFO HandlerThread:2123564 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:18:10,524 DEBUG HandlerThread:2123564 [system_info.py:_save_code():44] Saving code +2025-01-06 22:18:10,526 DEBUG HandlerThread:2123564 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:18:10,526 DEBUG HandlerThread:2123564 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:18:10,588 DEBUG HandlerThread:2123564 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:18:10,589 INFO HandlerThread:2123564 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:18:10,592 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:18:10,592 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:18:10,592 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:18:10,592 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:18:10,822 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:18:10,823 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:18:10,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:18:10,824 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/6tpjjiq2-wandb-metadata.json +2025-01-06 22:18:10,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:10,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:18:10,826 INFO HandlerThread:2123564 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/tb" + save: true +} +control { + relay_id: "cef7d876c4ad4fd199c5eb1a9bd32925" +} +_info { + stream_id: "917c47lf" +} + +2025-01-06 22:18:10,826 DEBUG HandlerThread:2123564 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:18:10,851 INFO wandb-upload_2:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/kdr9sx7r-diff.patch +2025-01-06 22:18:10,855 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/5sqz2j7o-code/FISH/eval_robot.py +2025-01-06 22:18:10,976 DEBUG SenderThread:2123564 [sender.py:send():379] send: telemetry +2025-01-06 22:18:10,976 DEBUG SenderThread:2123564 [sender.py:send():379] send: tbrecord +2025-01-06 22:18:10,977 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:18:10,977 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 with policy live +2025-01-06 22:18:11,156 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/v1x7seua-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:18:11,414 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/requirements.txt +2025-01-06 22:18:11,414 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:18:11,414 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/diff.patch +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-metadata.json +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/code/FISH/eval_robot.py +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/code +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/code/FISH +2025-01-06 22:18:11,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:12,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:13,413 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:13,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:14,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:15,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:15,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:16,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:17,522 DEBUG SenderThread:2123564 [sender.py:send():379] send: config +2025-01-06 22:18:17,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:18,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:19,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:19,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:20,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:20,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:21,416 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:21,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:22,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:23,416 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:23,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:24,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:25,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:18:25,823 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:18:25,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:25,972 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:26,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:27,417 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:27,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:28,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:29,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:30,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:30,979 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:31,418 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:31,826 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:32,826 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:33,419 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:33,826 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:34,826 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:35,420 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:35,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:36,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:36,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:37,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:38,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:39,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:40,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:18:40,823 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:18:40,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:41,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:41,930 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:42,422 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml +2025-01-06 22:18:42,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:43,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:44,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:45,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:46,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:47,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:47,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:48,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:49,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:50,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:51,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:52,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:52,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:53,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:54,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:55,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:18:55,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:18:55,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:56,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:57,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:57,962 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:58,829 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:59,829 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:00,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:01,662 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:19:01,662 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 with policy now +2025-01-06 22:19:01,662 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:19:01,663 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:19:01,663 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:19:01,664 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:19:01,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:01,901 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/37axuafr-media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media +2025-01-06 22:19:02,683 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/uc6srzzt-media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 +2025-01-06 22:19:02,837 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:03,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:03,674 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:03,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:04,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:05,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:06,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:07,429 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:07,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:08,833 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:08,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:09,429 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:09,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:10,507 DEBUG SystemMonitor:2123564 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:19:10,508 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:19:10,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:19:10,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:19:10,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:11,430 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:19:11,666 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/mzeyzwh7-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:19:11,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:12,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:13,431 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:13,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:14,346 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:14,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:15,432 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:15,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:16,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:17,432 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:17,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:18,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:19,829 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:19,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:20,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:21,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:22,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:23,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:24,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:25,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:19:25,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:19:25,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:25,905 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:26,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:27,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:28,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:29,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:30,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:31,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:31,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:32,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:33,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:34,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:35,438 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:35,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:36,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:36,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:37,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:38,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:39,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:40,508 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:19:40,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:19:40,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:19:40,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:41,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:42,088 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:42,576 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:19:42,577 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:19:42,577 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 with policy now +2025-01-06 22:19:42,577 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:19:42,577 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:19:42,578 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:19:42,791 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/qpwc9feh-media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 +2025-01-06 22:19:42,845 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:43,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:43,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:19:43,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 +2025-01-06 22:19:43,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:19:43,713 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/tnputsod-media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 +2025-01-06 22:19:43,847 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:44,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:45,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:45,845 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:46,845 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:47,442 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:47,830 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:47,845 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:48,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:49,443 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:49,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:50,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:51,225 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:19:51,226 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:19:51,226 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:19:51,443 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:51,444 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:19:51,444 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:19:51,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:52,851 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:53,444 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:53,637 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:53,852 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:54,852 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:55,445 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:55,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:19:55,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:19:55,847 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:56,848 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:57,446 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:57,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:58,789 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:58,861 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:59,446 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:59,863 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:00,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:01,863 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:02,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:03,830 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:03,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:04,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:05,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:06,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:07,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:08,837 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:08,864 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:09,864 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:10,509 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:20:10,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:20:10,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:20:10,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:11,864 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:12,867 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:13,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:13,974 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:14,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:15,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:16,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:17,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:18,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:19,834 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:19,866 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:20,868 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:21,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:22,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:23,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:24,868 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:24,882 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:25,180 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:20:25,180 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 with policy now +2025-01-06 22:20:25,182 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:20:25,183 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:20:25,183 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:20:25,183 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:20:25,397 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/xkkmrbaq-media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 +2025-01-06 22:20:25,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:25,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:20:25,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 +2025-01-06 22:20:25,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:20:25,728 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/u12h7cjr-media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 +2025-01-06 22:20:25,829 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:20:25,829 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:20:25,873 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:26,876 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:27,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:27,874 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:28,879 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:29,456 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:29,877 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:30,365 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:30,877 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:31,882 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:32,885 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:33,457 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:33,887 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:34,890 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:35,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:35,888 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:36,889 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:37,895 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:38,898 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:39,904 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:40,509 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:20:40,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:20:40,829 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:20:40,899 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:40,969 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:41,897 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:42,903 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:43,910 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:44,905 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:45,905 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:46,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:46,910 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:47,911 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:48,909 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:49,905 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:50,914 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:51,852 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:51,912 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:52,909 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:53,464 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:20:53,706 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/b8z1ithe-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:20:53,909 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:54,075 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:20:54,075 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:20:54,076 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:20:54,464 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:20:54,914 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:55,464 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:55,835 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:20:55,835 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:20:55,912 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:56,913 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:56,926 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:57,915 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:58,913 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:59,466 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:59,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:00,917 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:01,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:02,835 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:02,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:03,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:04,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:05,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:06,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:07,835 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:07,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:08,919 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:09,919 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:10,510 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:21:10,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:21:10,827 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:21:10,919 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:11,919 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:12,886 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:12,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:13,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:14,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:15,471 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:15,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:16,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:17,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:18,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:18,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:19,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:20,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:21,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:22,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:23,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:23,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:24,924 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:25,474 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:25,608 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:21:25,608 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 with policy now +2025-01-06 22:21:25,613 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:21:25,614 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:21:25,614 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:21:25,617 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:21:25,816 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/5k2hh8oi-media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 +2025-01-06 22:21:25,835 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:21:25,835 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:21:25,929 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:26,475 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:21:26,475 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 +2025-01-06 22:21:26,475 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:21:26,728 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/jpkayfdj-media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 +2025-01-06 22:21:26,935 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:27,475 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:27,930 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:28,928 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:28,933 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:29,932 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:30,936 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:31,477 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:31,936 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:32,935 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:33,477 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:33,938 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:34,121 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:34,937 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:35,941 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:36,934 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:37,943 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:38,946 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:39,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:39,949 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:40,511 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:21:40,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:21:40,844 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:21:40,946 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:41,948 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:42,135 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:21:42,135 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:21:42,135 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:21:42,481 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:21:42,481 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:21:42,948 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:43,481 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:43,971 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:44,966 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:45,384 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:45,952 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:46,952 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:47,482 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:47,952 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:48,956 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:49,955 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:50,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:50,953 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:51,954 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:52,953 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:53,953 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:54,961 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:55,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:21:55,828 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:21:55,962 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:55,979 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:56,962 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:57,486 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:57,965 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:58,962 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:59,963 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:00,963 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:01,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:01,963 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:02,964 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:03,964 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:04,964 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:05,968 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:06,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:06,966 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:07,966 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:08,965 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:09,965 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:10,511 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:22:10,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:22:10,828 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:22:10,967 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:11,490 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:11,880 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:11,967 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:12,968 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:13,976 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:14,598 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:22:14,598 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 with policy now +2025-01-06 22:22:14,607 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:22:14,608 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:14,608 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:14,609 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:14,811 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/ky50jorc-media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 +2025-01-06 22:22:14,982 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:15,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:15,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:15,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 +2025-01-06 22:22:15,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:22:15,758 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/ngi6k4n8-media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 +2025-01-06 22:22:15,986 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:16,986 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:17,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:17,850 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:17,982 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:19,006 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:19,493 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:19,987 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:20,994 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:21,494 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:21,999 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:22,851 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:22,997 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:23,495 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:24,006 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:25,009 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:25,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:22:25,841 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:22:25,994 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:27,006 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:27,975 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:27,999 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:29,000 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:30,016 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:31,005 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:32,012 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:33,004 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:33,058 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:22:33,058 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:22:33,265 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:22:33,498 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:33,498 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:33,499 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 22:22:33,499 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media +2025-01-06 22:22:33,499 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval +2025-01-06 22:22:33,499 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table +2025-01-06 22:22:33,510 INFO wandb-upload_1:2123564 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphynk_6q6 +2025-01-06 22:22:33,670 INFO wandb-upload_2:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/85pm559e-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 22:22:33,708 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:22:33,719 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:22:33,747 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/nelgnvoq-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:33,940 INFO SenderThread:2123564 [sender.py:send_request_log_artifact():1525] logged artifact run-917c47lf-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU3Mjg1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0MDMwNA==', 'latestArtifact': None}} +2025-01-06 22:22:33,940 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:22:33,940 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-06 22:22:33,941 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:33,941 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:33,941 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,941 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,942 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:33,942 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,946 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,946 DEBUG SenderThread:2123564 [sender.py:send():379] send: telemetry +2025-01-06 22:22:33,946 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,946 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,946 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,946 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,947 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,947 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,947 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,947 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,948 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,948 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,948 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,948 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,948 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,949 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,949 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,949 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,949 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,950 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,950 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,950 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,950 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,951 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,951 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:22:34,004 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:34,189 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/102le9fn-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 22:22:34,288 INFO wandb-upload_0:2123564 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmh_9g91f +2025-01-06 22:22:34,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:34,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:34,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json +2025-01-06 22:22:34,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval +2025-01-06 22:22:34,680 INFO SenderThread:2123564 [sender.py:send_request_log_artifact():1525] logged artifact run-917c47lf-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU3Mjg4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0MDMwOQ==', 'latestArtifact': None}} +2025-01-06 22:22:34,680 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:22:34,680 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json with policy now +2025-01-06 22:22:34,680 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:34,680 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:34,681 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:34,852 INFO wandb-upload_2:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/k8zaf9nu-media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json +2025-01-06 22:22:35,004 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:35,094 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/t645rmde-media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json +2025-01-06 22:22:35,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:35,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:36,005 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:36,361 DEBUG SenderThread:2123564 [sender.py:send():379] send: exit +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:send_exit():588] handling runtime: 265 +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:send_exit():594] send defer +2025-01-06 22:22:36,362 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,362 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:22:36,362 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:22:36,363 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,363 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:22:36,363 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:36,363 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:22:36,363 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:22:36,363 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,363 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:22:36,363 INFO HandlerThread:2123564 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:22:36,363 DEBUG SystemMonitor:2123564 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:22:36,363 DEBUG SystemMonitor:2123564 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:22:36,364 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:22:36,364 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:22:36,414 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:22:36,415 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:22:36,415 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:22:36,416 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:36,416 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:22:36,416 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:22:36,416 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:22:36,416 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,416 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:22:36,417 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:36,417 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:22:36,417 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:22:36,417 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,417 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:22:36,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:37,501 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:43,200 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:43,200 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:43,200 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:22:43,202 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:43,202 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:22:43,202 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:22:43,202 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:43,202 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:43,202 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:43,202 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:22:43,202 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:43,203 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:43,203 DEBUG SenderThread:2123564 [sender.py:send():379] send: summary +2025-01-06 22:22:43,207 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:43,207 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:43,207 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:22:43,207 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:22:43,207 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:43,207 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:22:43,208 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:43,208 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:22:43,211 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:43,328 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:22:43,328 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:43,328 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:22:43,329 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:43,329 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:22:43,363 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:43,503 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml +2025-01-06 22:22:43,503 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:46,639 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:22:46,639 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:46,640 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:46,640 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:22:46,640 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:46,640 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:22:46,640 INFO SenderThread:2123564 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:22:46,642 INFO SenderThread:2123564 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:22:46,644 INFO SenderThread:2123564 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:22:46,657 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:22:46,658 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:46,658 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:22:46,658 DEBUG SenderThread:2123564 [sender.py:send():379] send: artifact +2025-01-06 22:22:46,874 INFO SenderThread:2123564 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 22:22:46,874 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:46,874 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:22:46,874 INFO SenderThread:2123564 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:22:47,364 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:47,504 INFO SenderThread:2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:47,505 INFO SenderThread:2123564 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files +2025-01-06 22:22:47,505 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml config.yaml +2025-01-06 22:22:47,505 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json wandb-summary.json +2025-01-06 22:22:47,505 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:47,508 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/requirements.txt requirements.txt +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log output.log +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/diff.patch diff.patch +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 +2025-01-06 22:22:47,510 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 +2025-01-06 22:22:47,510 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 +2025-01-06 22:22:47,510 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 +2025-01-06 22:22:47,512 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 22:22:47,514 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json +2025-01-06 22:22:47,514 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:22:47,514 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:47,514 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:47,514 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:22:47,514 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:47,515 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:22:47,515 INFO SenderThread:2123564 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:22:47,681 INFO wandb-upload_2:2123564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml +2025-01-06 22:22:47,698 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:47,735 INFO wandb-upload_3:2123564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/requirements.txt +2025-01-06 22:22:47,742 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/nsvrz20o-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:47,744 INFO wandb-upload_4:2123564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:47,944 INFO Thread-11 (_thread_body):2123564 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:22:47,944 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:47,944 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:22:47,945 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:47,945 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:22:47,945 INFO SenderThread:2123564 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:22:47,946 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:22:47,946 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:47,946 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:22:47,946 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:47,946 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:22:47,946 INFO SenderThread:2123564 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:22:48,007 INFO SenderThread:2123564 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:22:48,007 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:22:48,007 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:48,007 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:22:48,007 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:48,007 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:22:48,007 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:22:48,007 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:48,007 DEBUG SenderThread:2123564 [sender.py:send():379] send: final +2025-01-06 22:22:48,008 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:22:48,008 DEBUG SenderThread:2123564 [sender.py:send():379] send: footer +2025-01-06 22:22:48,008 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:48,008 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:22:48,008 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:48,008 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:22:48,009 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:48,009 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:48,009 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:22:48,009 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:22:48,011 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:22:48,012 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:48,061 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:48,061 INFO MainThread:2123564 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:22:48,062 INFO MainThread:2123564 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:22:48,062 INFO MainThread:2123564 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:22:48,062 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:22:48,062 INFO HandlerThread:2123564 [handler.py:finish():882] shutting down handler +2025-01-06 22:22:49,010 INFO WriterThread:2123564 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/run-917c47lf.wandb +2025-01-06 22:22:49,061 INFO SenderThread:2123564 [sender.py:finish():1615] shutting down sender +2025-01-06 22:22:49,062 INFO SenderThread:2123564 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:22:49,062 INFO SenderThread:2123564 [file_pusher.py:join():175] waiting for file pusher diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug.log b/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..82ad1fc1326b77162e48852746096f1cf7a46859 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Configure stats pid to 2123316 +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/settings +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug.log +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug-internal.log +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:18:10,153 INFO MainThread:2123316 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf'} +2025-01-06 22:18:10,153 INFO MainThread:2123316 [wandb_init.py:init():619] starting backend +2025-01-06 22:18:10,153 INFO MainThread:2123316 [wandb_init.py:init():623] setting up manager +2025-01-06 22:18:10,158 INFO MainThread:2123316 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:18:10,159 INFO MainThread:2123316 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:18:10,172 INFO MainThread:2123316 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:18:10,178 INFO MainThread:2123316 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:18:10,428 INFO MainThread:2123316 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:18:10,494 INFO MainThread:2123316 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:18:10,494 INFO MainThread:2123316 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:18:10,823 INFO MainThread:2123316 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:18:10,823 INFO MainThread:2123316 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:18:10,823 INFO MainThread:2123316 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:18:10,823 INFO MainThread:2123316 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:18:10,825 INFO MainThread:2123316 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:18:10,825 INFO MainThread:2123316 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/tb, True +2025-01-06 22:18:17,522 INFO MainThread:2123316 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:22:49,064 WARNING MsgRouterThr:2123316 [router.py:message_loop():77] message_loop has been closed diff --git a/d4l0r92a/.hydra/config.yaml b/d4l0r92a/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/d4l0r92a/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/d4l0r92a/.hydra/hydra.yaml b/d4l0r92a/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6d8f5187f1f0c6666ff8e6defa21dd2be2fcd004 --- /dev/null +++ b/d4l0r92a/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/d4l0r92a/.hydra/overrides.yaml b/d4l0r92a/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/d4l0r92a/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/d4l0r92a/episode_rosbags/aligned_depth_to_color_K.npy b/d4l0r92a/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/d4l0r92a/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/d4l0r92a/episode_rosbags/cam_tf_world.npy b/d4l0r92a/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/d4l0r92a/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/d4l0r92a/episode_rosbags/color_K.npy b/d4l0r92a/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/d4l0r92a/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/d4l0r92a/episode_rosbags/depth_K.npy b/d4l0r92a/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/d4l0r92a/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/d4l0r92a/eval_robot.log b/d4l0r92a/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..e8b0aa436d2500019b17d8aae8c6cfdd0c28df96 --- /dev/null +++ b/d4l0r92a/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 17:28:41,553][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:28:41,556][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/d4l0r92a/eval_video/0_eval.mp4 b/d4l0r92a/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d5188f75868bf3a2945517025b9b07fc10b758d --- /dev/null +++ b/d4l0r92a/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d7712c707615309a216086ee73cfbee74224ba4ae5d90317e8c2ad9b3e19ad +size 1069145 diff --git a/d4l0r92a/eval_video/1_eval.mp4 b/d4l0r92a/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5ae81aecd2f322e46832f2735bc2da941c4cc0ba --- /dev/null +++ b/d4l0r92a/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92182b2f19c7840dbbc417ae96c831ccb23c05192a60892f7d2b7bb45ad210ff +size 1060722 diff --git a/d4l0r92a/eval_video/2_eval.mp4 b/d4l0r92a/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d0bbfd34757536c75624194814d3ffeb8f47b3c8 --- /dev/null +++ b/d4l0r92a/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6de2b4ebb693674e28cd406e28b2a8e1efeef585c27413662d0a62eb865a7c1 +size 1226861 diff --git a/d4l0r92a/eval_video/3_eval.mp4 b/d4l0r92a/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eee9268e195a3bcfc176eb35fc735f0d5f7a83e --- /dev/null +++ b/d4l0r92a/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cac182b4027a7521827c1102c32fd5c444c407326a8ad8df0f9138430fa4942c +size 989362 diff --git a/d4l0r92a/eval_video/4_eval.mp4 b/d4l0r92a/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5472cd6cd87e87f914ecf03cde4c7fff89e29d84 --- /dev/null +++ b/d4l0r92a/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12a3aaeba0fef5ed138d5576171025aa87960f11f4b9ef35a55af2046892acde +size 1240638 diff --git a/d4l0r92a/tb/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 b/d4l0r92a/tb/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 new file mode 100644 index 0000000000000000000000000000000000000000..0626c6a5d8d2613f292379b98097c61ebae04b5a --- /dev/null +++ b/d4l0r92a/tb/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edbe2335f3e5c3c66dced11b0431bb94bfb18f9be0807bcb06beaa49f4c9d2f0 +size 6519 diff --git a/d4l0r92a/wandb/debug-internal.log b/d4l0r92a/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6900f99251bdea9ea4c1dc6f49affded98e65eef --- /dev/null +++ b/d4l0r92a/wandb/debug-internal.log @@ -0,0 +1,765 @@ +2025-01-06 17:28:47,455 INFO StreamThr :2090663 [internal.py:wandb_internal():85] W&B internal server running at pid: 2090663, started at: 2025-01-06 17:28:47.455096 +2025-01-06 17:28:47,456 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:28:47,468 INFO WriterThread:2090663 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/run-d4l0r92a.wandb +2025-01-06 17:28:47,468 DEBUG SenderThread:2090663 [sender.py:send():379] send: header +2025-01-06 17:28:47,477 DEBUG SenderThread:2090663 [sender.py:send():379] send: run +2025-01-06 17:28:47,747 INFO SenderThread:2090663 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files +2025-01-06 17:28:47,747 INFO SenderThread:2090663 [sender.py:_start_run_threads():1188] run started: d4l0r92a with start time 1736202527.45553 +2025-01-06 17:28:47,763 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:28:47,763 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:28:47,839 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:28:47,843 DEBUG HandlerThread:2090663 [system_info.py:__init__():26] System info init +2025-01-06 17:28:47,844 DEBUG HandlerThread:2090663 [system_info.py:__init__():41] System info init done +2025-01-06 17:28:47,844 INFO HandlerThread:2090663 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:28:47,844 INFO SystemMonitor:2090663 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:28:47,844 INFO HandlerThread:2090663 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:28:47,844 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:28:47,844 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:28:47,845 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:28:47,845 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:28:47,846 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:28:47,856 DEBUG HandlerThread:2090663 [system_info.py:probe():152] Probing system +2025-01-06 17:28:47,858 DEBUG HandlerThread:2090663 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:28:47,863 DEBUG HandlerThread:2090663 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:28:47,863 DEBUG HandlerThread:2090663 [system_info.py:probe():200] Probing system done +2025-01-06 17:28:47,863 DEBUG HandlerThread:2090663 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:28:47.856980', 'startedAt': '2025-01-06T22:28:47.447626', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.221375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 340.60540771484375}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:28:47,863 INFO HandlerThread:2090663 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:28:47,863 INFO HandlerThread:2090663 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:28:47,863 DEBUG HandlerThread:2090663 [system_info.py:_save_code():44] Saving code +2025-01-06 17:28:47,864 DEBUG HandlerThread:2090663 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:28:47,864 DEBUG HandlerThread:2090663 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:28:47,929 DEBUG HandlerThread:2090663 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:28:47,929 INFO HandlerThread:2090663 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:28:47,934 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:28:47,935 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:28:47,935 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:28:47,935 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:28:48,174 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/h3n6188w-wandb-metadata.json +2025-01-06 17:28:48,187 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:28:48,187 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:28:48,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:28:48,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:28:48,206 INFO wandb-upload_1:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/h5szt28m-code/FISH/eval_robot.py +2025-01-06 17:28:48,207 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/lqn5ijm1-diff.patch +2025-01-06 17:28:48,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:48,233 INFO HandlerThread:2090663 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/tb" + save: true +} +control { + relay_id: "53cad69e1a284cc0844ac292d4426ea7" +} +_info { + stream_id: "d4l0r92a" +} + +2025-01-06 17:28:48,233 DEBUG HandlerThread:2090663 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:28:48,263 DEBUG SenderThread:2090663 [sender.py:send():379] send: telemetry +2025-01-06 17:28:48,284 DEBUG SenderThread:2090663 [sender.py:send():379] send: tbrecord +2025-01-06 17:28:48,285 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:28:48,285 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 with policy live +2025-01-06 17:28:48,471 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/dbm412tw-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:28:48,749 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/requirements.txt +2025-01-06 17:28:48,749 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-metadata.json +2025-01-06 17:28:48,749 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:28:48,749 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:28:48,750 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/diff.patch +2025-01-06 17:28:48,750 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code/FISH/eval_robot.py +2025-01-06 17:28:48,750 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code +2025-01-06 17:28:48,750 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code/FISH +2025-01-06 17:28:49,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:50,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:50,748 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:28:51,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:52,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:53,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:28:53,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:54,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:55,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:56,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:56,787 DEBUG SenderThread:2090663 [sender.py:send():379] send: config +2025-01-06 17:28:57,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:58,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:28:58,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:58,751 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:28:59,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:00,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:00,751 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:01,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:02,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:02,752 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:03,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:29:03,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:29:03,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:03,324 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:04,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:05,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:06,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:06,753 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:07,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:08,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:09,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:09,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:10,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:10,754 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:11,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:12,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:12,755 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:13,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:14,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:14,393 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:14,755 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:15,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:16,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:17,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:18,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:29:18,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:29:18,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:19,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:19,861 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:20,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:20,757 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml +2025-01-06 17:29:20,757 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:21,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:22,205 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:23,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:24,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:24,758 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:25,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:25,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:26,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:27,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:28,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:29,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:30,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:31,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:31,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:32,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:33,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:29:33,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:29:33,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:34,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:35,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:36,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:36,265 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:37,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:37,445 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:29:37,445 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 with policy now +2025-01-06 17:29:37,445 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:29:37,446 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:29:37,447 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:29:37,447 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:29:37,702 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/5j3gsj46-media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 +2025-01-06 17:29:37,761 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 +2025-01-06 17:29:37,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:29:37,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos +2025-01-06 17:29:37,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media +2025-01-06 17:29:37,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:29:38,016 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/tstaoxft-media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 +2025-01-06 17:29:38,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:38,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:39,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:40,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:41,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:41,597 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:42,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:42,763 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:43,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:44,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:44,764 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:45,196 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:46,196 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:47,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:47,306 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:47,846 DEBUG SystemMonitor:2090663 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:29:47,847 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:29:48,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:29:48,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:29:48,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:49,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:50,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:51,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:52,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:52,309 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:53,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:54,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:55,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:56,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:57,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:58,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:58,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:59,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:00,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:01,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:02,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:02,770 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:30:03,012 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/gcd6wjds-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:30:03,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:30:03,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:30:03,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:03,261 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:04,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:04,770 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:05,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:06,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:07,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:08,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:08,772 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:09,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:09,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:10,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:11,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:12,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:13,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:14,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:15,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:15,201 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:16,201 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:17,201 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:17,847 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:30:18,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:30:18,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:30:18,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:19,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:20,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:20,773 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:20,775 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:21,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:22,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:23,203 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:24,203 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:24,777 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:25,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:26,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:26,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:27,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:28,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:29,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:30,090 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:30:30,090 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 with policy now +2025-01-06 17:30:30,091 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:30:30,091 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:30:30,091 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:30:30,093 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:30:30,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:30,335 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/6qglpwso-media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 +2025-01-06 17:30:30,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:30:30,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:30,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 +2025-01-06 17:30:30,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:30:31,058 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/i569hmqq-media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 +2025-01-06 17:30:31,208 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:32,101 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:32,206 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:32,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:33,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:30:33,189 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:30:33,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:34,216 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:34,780 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:35,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:36,209 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:37,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:38,065 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:38,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:38,781 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:39,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:40,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:41,212 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:42,211 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:43,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:43,211 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:44,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:45,212 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:46,211 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:47,211 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:47,847 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:30:48,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:30:48,191 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:30:48,237 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:48,316 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:49,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:50,212 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:51,212 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:52,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:53,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:53,602 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:53,786 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:30:54,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:54,736 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:30:54,736 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:30:54,737 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:30:54,786 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:30:54,787 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:55,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:56,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:56,787 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:57,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:58,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:58,788 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:59,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:59,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:00,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:01,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:02,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:03,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:31:03,189 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:31:03,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:04,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:04,319 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:05,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:06,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:07,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:08,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:09,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:10,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:10,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:11,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:12,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:13,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:14,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:15,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:16,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:16,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:17,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:17,848 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:31:18,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:31:18,189 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:31:18,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:19,216 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:20,216 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:21,217 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:21,281 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:22,216 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:23,217 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:24,217 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:25,219 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:25,829 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:31:25,829 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 with policy now +2025-01-06 17:31:25,829 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:31:25,830 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:31:25,830 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:31:25,831 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:31:26,091 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/fwfcg65b-media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 +2025-01-06 17:31:26,223 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:26,797 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:31:26,797 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:26,798 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 +2025-01-06 17:31:26,798 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:31:26,851 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:27,096 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/gmh3byrr-media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 +2025-01-06 17:31:27,219 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:28,226 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:29,221 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:30,226 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:30,799 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:31,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:31,865 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:32,224 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:32,799 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:33,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:31:33,198 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:31:33,245 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:34,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:35,227 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:36,230 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:37,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:37,324 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:37,802 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:31:38,047 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/heuqkeu5-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:31:38,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:38,493 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:31:38,493 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:31:38,494 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:31:38,802 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:31:38,803 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:39,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:40,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:40,804 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:41,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:42,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:42,805 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:42,971 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:43,230 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:44,230 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:45,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:46,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:47,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:47,848 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:31:48,108 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:48,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:31:48,189 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:31:48,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:49,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:50,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:50,808 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:51,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:52,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:53,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:53,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:54,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:55,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:56,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:56,810 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:57,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:58,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:58,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:59,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:00,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:01,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:02,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:03,196 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:32:03,196 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:32:03,235 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:03,306 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:03,760 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:32:03,760 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 with policy now +2025-01-06 17:32:03,761 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:32:03,763 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:32:03,763 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:32:03,764 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:32:03,814 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:32:03,814 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 +2025-01-06 17:32:03,814 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:32:04,011 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/1buyzstt-media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 +2025-01-06 17:32:04,234 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:04,262 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/a4v633zo-media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 +2025-01-06 17:32:04,814 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:05,247 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:06,237 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:07,240 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:08,242 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:08,816 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:09,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:09,244 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:10,241 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:10,816 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:11,245 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:12,241 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:13,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:14,245 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:15,208 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:15,246 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:16,247 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:17,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:17,849 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:32:18,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:32:18,197 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:32:18,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:19,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:20,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:20,263 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:21,257 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:22,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:23,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:23,547 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:32:23,547 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:32:23,547 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:32:23,820 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:32:23,821 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:32:24,255 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:24,821 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:25,258 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:25,501 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:26,256 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:26,822 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:27,289 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:28,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:28,822 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:29,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:30,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:31,207 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:31,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:32,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:33,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:32:33,190 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:32:33,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:34,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:34,824 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:35,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:36,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:36,853 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:37,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:38,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:39,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:40,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:41,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:42,207 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:42,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:43,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:44,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:45,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:46,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:47,208 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:47,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:47,850 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:32:48,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:32:48,190 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:32:48,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:49,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:50,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:51,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:52,255 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:52,342 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:53,255 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:54,255 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:55,273 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:55,467 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:32:55,467 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 with policy now +2025-01-06 17:32:55,470 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:32:55,470 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:32:55,470 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:32:55,471 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:32:55,723 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/k6jof3hy-media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 +2025-01-06 17:32:55,831 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:32:55,832 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 +2025-01-06 17:32:55,832 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:32:56,096 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/2dsre9dk-media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 +2025-01-06 17:32:56,261 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:56,832 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:57,264 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:57,501 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:58,268 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:59,271 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:00,271 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:00,833 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:01,287 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:02,299 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:02,834 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:03,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:33:03,198 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:33:03,266 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:03,276 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:03,834 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:04,076 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/jzc7jh33-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:04,272 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:04,599 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:33:04,599 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:04,599 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:04,654 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:33:04,655 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table +2025-01-06 17:33:04,837 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:33:05,023 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/ev91wtea-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:33:05,120 INFO wandb-upload_2:2090663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplp0yrt7a +2025-01-06 17:33:05,181 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:33:05,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:33:05,272 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:05,594 INFO SenderThread:2090663 [sender.py:send_request_log_artifact():1525] logged artifact run-d4l0r92a-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjExMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMjQ2OQ==', 'latestArtifact': None}} +2025-01-06 17:33:05,594 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:33:05,594 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-06 17:33:05,595 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:33:05,595 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,596 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,597 DEBUG SenderThread:2090663 [sender.py:send():379] send: telemetry +2025-01-06 17:33:05,597 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,597 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,597 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,597 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,597 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,598 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,598 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,598 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,598 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,601 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,603 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,604 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,604 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,604 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,604 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,606 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,606 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,606 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,607 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,607 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,607 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,607 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,607 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:33:05,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:05,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:05,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-06 17:33:05,836 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval +2025-01-06 17:33:05,851 INFO wandb-upload_1:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/jwr53dwx-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:33:06,005 INFO wandb-upload_2:2090663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpylcyyb7s +2025-01-06 17:33:06,033 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/rz253b0p-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-06 17:33:06,272 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:06,389 INFO SenderThread:2090663 [sender.py:send_request_log_artifact():1525] logged artifact run-d4l0r92a-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjE0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMjQ3Mg==', 'latestArtifact': None}} +2025-01-06 17:33:06,389 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:33:06,390 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json with policy now +2025-01-06 17:33:06,390 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:33:06,390 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:06,391 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:06,638 INFO wandb-upload_1:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/x296pkve-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-06 17:33:06,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:06,836 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:07,272 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:07,994 DEBUG SenderThread:2090663 [sender.py:send():379] send: exit +2025-01-06 17:33:07,994 INFO SenderThread:2090663 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:33:07,994 INFO SenderThread:2090663 [sender.py:send_exit():588] handling runtime: 260 +2025-01-06 17:33:07,994 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:07,994 INFO SenderThread:2090663 [sender.py:send_exit():594] send defer +2025-01-06 17:33:07,994 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:07,994 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:33:07,995 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:07,995 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:33:07,995 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:33:07,995 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:07,995 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:33:07,995 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:07,995 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:33:07,995 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:33:07,995 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:07,995 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:33:07,995 INFO HandlerThread:2090663 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:33:07,995 DEBUG SystemMonitor:2090663 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:33:07,995 DEBUG SystemMonitor:2090663 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:33:07,997 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:33:07,997 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:33:08,050 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:33:08,050 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:33:08,051 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:33:08,051 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:08,051 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:33:08,051 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:33:08,051 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:33:08,051 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:08,051 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:33:08,052 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:08,052 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:33:08,052 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:33:08,052 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:08,052 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:33:08,836 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:08,837 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:09,837 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml +2025-01-06 17:33:14,604 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:14,605 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:14,605 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:33:14,605 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:14,606 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:14,606 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:33:14,606 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:33:14,606 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:14,606 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:33:14,606 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:14,606 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:14,607 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:33:14,607 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:14,607 DEBUG SenderThread:2090663 [sender.py:send():379] send: summary +2025-01-06 17:33:14,611 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:14,611 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:14,611 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:33:14,611 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:33:14,611 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:14,611 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:33:14,612 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:14,612 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:33:14,612 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:33:14,612 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:14,612 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:14,612 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:33:14,612 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:14,612 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:33:14,839 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:14,995 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:17,998 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:33:17,998 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:17,998 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:17,998 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:33:17,999 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:17,999 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:33:17,999 INFO SenderThread:2090663 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:33:18,001 INFO SenderThread:2090663 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:33:18,003 INFO SenderThread:2090663 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:33:18,013 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:33:18,013 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:18,013 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:33:18,014 DEBUG SenderThread:2090663 [sender.py:send():379] send: artifact +2025-01-06 17:33:18,387 INFO wandb-upload_2:2090663 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp30y1vr2u +2025-01-06 17:33:18,388 INFO wandb-upload_1:2090663 [upload_job.py:push():85] Skipped uploading /tmp/tmpelzxd0ux/wandb-job.json +2025-01-06 17:33:18,481 INFO wandb-upload_0:2090663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf94pqpss +2025-01-06 17:33:18,840 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:18,943 INFO SenderThread:2090663 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'versionIndex': 423}}} +2025-01-06 17:33:18,943 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:18,943 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:33:18,943 INFO SenderThread:2090663 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:33:18,996 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:19,841 INFO SenderThread:2090663 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files +2025-01-06 17:33:19,841 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml config.yaml +2025-01-06 17:33:19,842 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json wandb-summary.json +2025-01-06 17:33:19,842 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/requirements.txt requirements.txt +2025-01-06 17:33:19,842 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:33:19,842 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:19,843 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log output.log +2025-01-06 17:33:19,844 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/diff.patch diff.patch +2025-01-06 17:33:19,849 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:33:19,850 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:19,851 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:19,851 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:33:19,851 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:19,853 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:19,853 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:33:19,854 INFO SenderThread:2090663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:33:19,996 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:19,997 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:20,033 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml +2025-01-06 17:33:20,036 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:20,088 INFO wandb-upload_1:2090663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/requirements.txt +2025-01-06 17:33:20,109 INFO wandb-upload_3:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/sot7vj7s-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:20,117 INFO wandb-upload_4:2090663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:20,318 INFO Thread-11 (_thread_body):2090663 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:33:20,318 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:20,318 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:33:20,318 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:20,318 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:33:20,318 INFO SenderThread:2090663 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:33:20,319 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:33:20,319 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:20,320 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:33:20,320 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:20,320 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:33:20,320 INFO SenderThread:2090663 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:33:20,390 INFO SenderThread:2090663 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:33:20,390 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:33:20,390 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:20,391 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:33:20,391 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:20,391 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:33:20,391 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:33:20,391 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:20,391 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:33:20,391 DEBUG SenderThread:2090663 [sender.py:send():379] send: final +2025-01-06 17:33:20,391 DEBUG SenderThread:2090663 [sender.py:send():379] send: footer +2025-01-06 17:33:20,391 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:20,391 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:33:20,392 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:20,392 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:33:20,392 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:20,392 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:33:20,393 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:20,393 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:33:20,393 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:33:20,395 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:20,453 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:20,454 INFO MainThread:2090663 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:33:20,454 INFO MainThread:2090663 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:33:20,455 INFO MainThread:2090663 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:33:20,455 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:33:20,455 INFO HandlerThread:2090663 [handler.py:finish():882] shutting down handler +2025-01-06 17:33:21,393 INFO WriterThread:2090663 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/run-d4l0r92a.wandb +2025-01-06 17:33:21,453 INFO SenderThread:2090663 [sender.py:finish():1615] shutting down sender +2025-01-06 17:33:21,454 INFO SenderThread:2090663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:33:21,454 INFO SenderThread:2090663 [file_pusher.py:join():175] waiting for file pusher diff --git a/d4l0r92a/wandb/debug.log b/d4l0r92a/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..517f79c4d32ee10b3e15d8d9ce2731f0ec1040c3 --- /dev/null +++ b/d4l0r92a/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Configure stats pid to 2090492 +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/settings +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug.log +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug-internal.log +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a'} +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:init():619] starting backend +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:init():623] setting up manager +2025-01-06 17:28:47,454 INFO MainThread:2090492 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:28:47,455 INFO MainThread:2090492 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:28:47,468 INFO MainThread:2090492 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:28:47,475 INFO MainThread:2090492 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:28:47,762 INFO MainThread:2090492 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:28:47,833 INFO MainThread:2090492 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:28:47,833 INFO MainThread:2090492 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:28:48,188 INFO MainThread:2090492 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:28:48,188 INFO MainThread:2090492 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:28:48,188 INFO MainThread:2090492 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:28:48,188 INFO MainThread:2090492 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:28:48,190 INFO MainThread:2090492 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:28:48,190 INFO MainThread:2090492 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/tb, True +2025-01-06 17:28:56,786 INFO MainThread:2090492 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:33:21,455 WARNING MsgRouterThr:2090492 [router.py:message_loop():77] message_loop has been closed diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code/FISH/eval_robot.py b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..315117a62b4177672b22663b64d1e1e9c63f915b --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736202527 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/diff.patch b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c28171b0ccbf1f8e5d0d27b96acf339a03de1c06 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..a719175 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0ada6cde6d32ac525d48de8076e15c8904b601f9 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[0, 5, 0.0, 0, 0.5534158944976787, 0, 0.515846252438237, 0, 0.3141024113073212, 0, 0.3513819575606989]]} \ No newline at end of file diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..7d5188f75868bf3a2945517025b9b07fc10b758d --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80d7712c707615309a216086ee73cfbee74224ba4ae5d90317e8c2ad9b3e19ad +size 1069145 diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5ae81aecd2f322e46832f2735bc2da941c4cc0ba --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92182b2f19c7840dbbc417ae96c831ccb23c05192a60892f7d2b7bb45ad210ff +size 1060722 diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d0bbfd34757536c75624194814d3ffeb8f47b3c8 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6de2b4ebb693674e28cd406e28b2a8e1efeef585c27413662d0a62eb865a7c1 +size 1226861 diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eee9268e195a3bcfc176eb35fc735f0d5f7a83e --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cac182b4027a7521827c1102c32fd5c444c407326a8ad8df0f9138430fa4942c +size 989362 diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5472cd6cd87e87f914ecf03cde4c7fff89e29d84 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12a3aaeba0fef5ed138d5576171025aa87960f11f4b9ef35a55af2046892acde +size 1240638 diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..bcc0847aec3123ccecdc5d1eecefb04fdfc95038 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log @@ -0,0 +1,280 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736202536.787156]: resetting environment +[INFO] [1736202536.790302]: cleared current plan +[INFO] [1736202536.791235]: moving to home +[INFO] [1736202537.794976]: reached home +[INFO] [1736202537.796308]: reset action history +[INFO] [1736202540.505958]: environment reset +Starting episode 0 +[INFO] [1736202540.508176]: resetting environment +[INFO] [1736202540.511361]: cleared current plan +[INFO] [1736202540.512139]: moving to home +[INFO] [1736202541.513953]: reached home +[INFO] [1736202541.515133]: reset action history +[INFO] [1736202544.272377]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736202549.346481]: resetting environment +[INFO] [1736202549.352781]: cleared current plan +[INFO] [1736202549.354738]: moving to home +[INFO] [1736202550.357229]: reached home +[INFO] [1736202550.358905]: reset action history +[INFO] [1736202553.067190]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/episode_rosbags/episode_0_2025-01-06-17-29-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2090492.log +started roslaunch server http://158.130.50.37:34899/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2090789] +process[print_text-2]: started with pid [2090790] +process[rosbag_record-3]: started with pid [2090791] +started bagging! +For topic gripper_width: timestamp difference is 115306109 for nearest: 1736202558806792944 - target: 1736202558691486835 at idx 83 +For topic gripper_width: timestamp difference is -217197907 for nearest: 1736202561840126026 - target: 1736202562057323933 at idx 130 +preempting episode +[INFO] [1736202576.514235]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736202577.235354]: Stored episode 1. +[INFO] [1736202577.236912]: Saving video... +[INFO] [1736202577.432134]: Video saved! +Starting episode 1 +[INFO] [1736202577.446111]: resetting environment +[INFO] [1736202577.450266]: cleared current plan +[INFO] [1736202577.450516]: moving to home +[INFO] [1736202580.553189]: reached home +[INFO] [1736202580.554895]: reset action history +[INFO] [1736202583.261666]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202602.229553]: resetting environment +[INFO] [1736202602.231676]: cleared current plan +[INFO] [1736202602.231834]: moving to home +[INFO] [1736202603.233269]: reached home +[INFO] [1736202603.234035]: reset action history +[INFO] [1736202605.946727]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/episode_rosbags/episode_1_2025-01-06-17-30-05.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2090492.log +started roslaunch server http://158.130.50.37:34351/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2090904] +process[print_text-5]: started with pid [2090905] +process[rosbag_record-6]: started with pid [2090906] +started bagging! +For topic gripper_width: timestamp difference is -112777544 for nearest: 1736202618540126013 - target: 1736202618652903557 at idx 176 +For topic gripper_width: timestamp difference is -104115820 for nearest: 1736202622740126753 - target: 1736202622844242573 at idx 234 +For topic gripper_width: timestamp difference is -303677654 for nearest: 1736202622740126753 - target: 1736202623043804407 at idx 234 +preempting episode +[INFO] [1736202628.644033]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736202629.366561]: Stored episode 2. +[INFO] [1736202629.366961]: Saving video... +[WARN] [1736202629.842725]: Plan exhausted +[WARN] [1736202629.883267]: Plan exhausted +[WARN] [1736202629.934382]: Plan exhausted +[WARN] [1736202629.963627]: Plan exhausted +[WARN] [1736202630.003019]: Plan exhausted +[WARN] [1736202630.054711]: Plan exhausted +[WARN] [1736202630.085013]: Plan exhausted +[INFO] [1736202630.057576]: Video saved! +Starting episode 2 +[INFO] [1736202630.093055]: resetting environment +[INFO] [1736202630.100246]: cleared current plan +[INFO] [1736202630.100672]: moving to home +[INFO] [1736202633.304625]: reached home +[INFO] [1736202633.305545]: reset action history +[INFO] [1736202636.022165]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202653.629515]: resetting environment +[INFO] [1736202653.642074]: cleared current plan +[INFO] [1736202653.648282]: moving to home +[INFO] [1736202654.649824]: reached home +[INFO] [1736202654.650499]: reset action history +[INFO] [1736202657.358112]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/episode_rosbags/episode_2_2025-01-06-17-30-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2090492.log +started roslaunch server http://158.130.50.37:36719/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2091066] +process[print_text-8]: started with pid [2091067] +process[rosbag_record-9]: started with pid [2091068] +started bagging! +[INFO] [1736202684.552390]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736202685.338166]: Stored episode 3. +[INFO] [1736202685.343653]: Saving video... +[INFO] [1736202685.797060]: Video saved! +Starting episode 3 +[INFO] [1736202685.830728]: resetting environment +[INFO] [1736202685.849307]: cleared current plan +[INFO] [1736202685.850487]: moving to home +[INFO] [1736202689.061076]: reached home +[INFO] [1736202689.065712]: reset action history +[INFO] [1736202691.864394]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202697.575299]: resetting environment +[INFO] [1736202697.583732]: cleared current plan +[INFO] [1736202697.585711]: moving to home +[INFO] [1736202698.588843]: reached home +[INFO] [1736202698.593336]: reset action history +[INFO] [1736202701.475934]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/episode_rosbags/episode_3_2025-01-06-17-31-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2090492.log +started roslaunch server http://158.130.50.37:42257/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2091199] +process[print_text-11]: started with pid [2091200] +process[rosbag_record-12]: started with pid [2091201] +started bagging! +For topic gripper_width: timestamp difference is 100708954 for nearest: 1736202708106792204 - target: 1736202708006083250 at idx 98 +For topic gripper_width: timestamp difference is 101469396 for nearest: 1736202709106742977 - target: 1736202709005273581 at idx 115 +For topic gripper_width: timestamp difference is 102157956 for nearest: 1736202709906793242 - target: 1736202709804635286 at idx 128 +For topic gripper_width: timestamp difference is 106040421 for nearest: 1736202714706811133 - target: 1736202714600770712 at idx 194 +preempting episode[INFO] [1736202722.672354]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736202723.449343]: Stored episode 4. +[INFO] [1736202723.450733]: Saving video... +[INFO] [1736202723.717128]: Video saved! +Starting episode 4 +[INFO] [1736202723.773055]: resetting environment +[INFO] [1736202723.787654]: cleared current plan +[INFO] [1736202723.789386]: moving to home +[INFO] [1736202726.792202]: reached home +[INFO] [1736202726.793717]: reset action history +[INFO] [1736202729.587070]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202743.416418]: resetting environment +[INFO] [1736202743.448870]: cleared current plan +[INFO] [1736202743.453988]: moving to home +[INFO] [1736202744.459445]: reached home +[INFO] [1736202744.462914]: reset action history +[INFO] [1736202747.242874]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/episode_rosbags/episode_4_2025-01-06-17-32-27.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2090492.log +started roslaunch server http://158.130.50.37:32835/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2091336] +process[print_text-14]: started with pid [2091337] +process[rosbag_record-15]: started with pid [2091338] +started bagging! +For topic gripper_width: timestamp difference is 104331665 for nearest: 1736202753806788378 - target: 1736202753702456713 at idx 102 +[INFO] [1736202774.353632]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736202775.139909]: Stored episode 5. +[INFO] [1736202775.148676]: Saving video... +[INFO] [1736202775.414556]: Video saved! +[INFO] [1736202775.470753]: resetting environment +[INFO] [1736202775.493368]: cleared current plan +[INFO] [1736202775.500090]: moving to home +[INFO] [1736202778.515300]: reached home +[INFO] [1736202778.520668]: reset action history +[INFO] [1736202781.314612]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/requirements.txt b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-metadata.json b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c134a01a6f8d863a7bacafc14cabcf3dbe72fa60 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-06T22:28:47.856980", + "startedAt": "2025-01-06T22:28:47.447626", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3749.221375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.996, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 340.60540771484375 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..90c148b295bb55bbb3e1e937005143aabd1a6b8c --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "80d7712c707615309a216086ee73cfbee74224ba4ae5d90317e8c2ad9b3e19ad", "size": 1069145, "path": "media/videos/eval/0_eval_0_80d7712c707615309a21.mp4"}, "global_step": 595, "_timestamp": 1736202785.1085122, "_runtime": 257.65298223495483, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "92182b2f19c7840dbbc417ae96c831ccb23c05192a60892f7d2b7bb45ad210ff", "size": 1060722, "path": "media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4"}, "eval/num_success": 0.0, "episode": 5.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "c6de2b4ebb693674e28cd406e28b2a8e1efeef585c27413662d0a62eb865a7c1", "size": 1226861, "path": "media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "cac182b4027a7521827c1102c32fd5c444c407326a8ad8df0f9138430fa4942c", "size": 989362, "path": "media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "12a3aaeba0fef5ed138d5576171025aa87960f11f4b9ef35a55af2046892acde", "size": 1240638, "path": "media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://4iwpifdujxvk9dshj1jk42c6v4dadkxf4chinlsh3oo16681ry69mknnykibuo8z4l610b77tyv1h7trrju0dmnjwr4i9ktiobvyymfnicih50qgebcg8a3j6bzo7sra/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://cya7hq2g3mojb7x28kmcxewvikvghy0waugswrpb3g9z1y03eg6mdnqf5fwz4v5k5gyofpl2hbwe9hjqcglf2ttts2jrzknnhxmvp041zdiays3dm81806twh346557k:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 0, "total_num_episodes": 5, "total_success_rate": 0.0, "total_success_rate_lb_0.01": 0, "total_success_rate_ub_0.01": 0.5534158944976787, "total_success_rate_lb_0.025": 0, "total_success_rate_ub_0.025": 0.515846252438237, "total_success_rate_lb_0.05": 0, "total_success_rate_ub_0.05": 0.3141024113073212, "total_success_rate_lb_0.1": 0, "total_success_rate_ub_0.1": 0.3513819575606989, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "3fbfe773b6a6adb9e9f84231abc953c71bfff9eaa3d69b2bf95911c7d2f3a912", "size": 434, "artifact_path": "wandb-client-artifact://9tgeggt5pw36ka5vh7vrhuop8bzormyvtxs22n06unnpd68q8a3d0rsl12bdymjpabwzerkln7ezqyuhi5qfd7lfrh6pi7klriej8qb300q6czasvw30t9qbj0x287an/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://hsdkpmv34oduebodhstmqejaonbw8qav3g9qa34g9y55t9ilyrp31o3nln6qh2207yb7ovmxvo91psu90a7ysb0fw9n2n045k99q40gnrafw0h4e5h6crq7afzoftbq8:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 260}, "eval/total_success_rate": 0.0, "eval/total_success_rate_lb_0.01": 0.0, "eval/total_success_rate_ub_0.01": 0.5534158945083618, "eval/total_success_rate_lb_0.025": 0.0, "eval/total_success_rate_ub_0.025": 0.5158462524414062, "eval/total_success_rate_lb_0.05": 0.0, "eval/total_success_rate_ub_0.05": 0.3141024112701416, "eval/total_success_rate_lb_0.1": 0.0, "eval/total_success_rate_ub_0.1": 0.35138195753097534} \ No newline at end of file diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug-internal.log b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6900f99251bdea9ea4c1dc6f49affded98e65eef --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug-internal.log @@ -0,0 +1,765 @@ +2025-01-06 17:28:47,455 INFO StreamThr :2090663 [internal.py:wandb_internal():85] W&B internal server running at pid: 2090663, started at: 2025-01-06 17:28:47.455096 +2025-01-06 17:28:47,456 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:28:47,468 INFO WriterThread:2090663 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/run-d4l0r92a.wandb +2025-01-06 17:28:47,468 DEBUG SenderThread:2090663 [sender.py:send():379] send: header +2025-01-06 17:28:47,477 DEBUG SenderThread:2090663 [sender.py:send():379] send: run +2025-01-06 17:28:47,747 INFO SenderThread:2090663 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files +2025-01-06 17:28:47,747 INFO SenderThread:2090663 [sender.py:_start_run_threads():1188] run started: d4l0r92a with start time 1736202527.45553 +2025-01-06 17:28:47,763 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:28:47,763 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:28:47,839 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:28:47,843 DEBUG HandlerThread:2090663 [system_info.py:__init__():26] System info init +2025-01-06 17:28:47,844 DEBUG HandlerThread:2090663 [system_info.py:__init__():41] System info init done +2025-01-06 17:28:47,844 INFO HandlerThread:2090663 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:28:47,844 INFO SystemMonitor:2090663 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:28:47,844 INFO HandlerThread:2090663 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:28:47,844 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:28:47,844 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:28:47,845 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:28:47,845 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:28:47,846 INFO SystemMonitor:2090663 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:28:47,856 DEBUG HandlerThread:2090663 [system_info.py:probe():152] Probing system +2025-01-06 17:28:47,858 DEBUG HandlerThread:2090663 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:28:47,863 DEBUG HandlerThread:2090663 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:28:47,863 DEBUG HandlerThread:2090663 [system_info.py:probe():200] Probing system done +2025-01-06 17:28:47,863 DEBUG HandlerThread:2090663 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:28:47.856980', 'startedAt': '2025-01-06T22:28:47.447626', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.221375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.996, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 340.60540771484375}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:28:47,863 INFO HandlerThread:2090663 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:28:47,863 INFO HandlerThread:2090663 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:28:47,863 DEBUG HandlerThread:2090663 [system_info.py:_save_code():44] Saving code +2025-01-06 17:28:47,864 DEBUG HandlerThread:2090663 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:28:47,864 DEBUG HandlerThread:2090663 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:28:47,929 DEBUG HandlerThread:2090663 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:28:47,929 INFO HandlerThread:2090663 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:28:47,934 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:28:47,935 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:28:47,935 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:28:47,935 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:28:48,174 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/h3n6188w-wandb-metadata.json +2025-01-06 17:28:48,187 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:28:48,187 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:28:48,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:28:48,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:28:48,206 INFO wandb-upload_1:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/h5szt28m-code/FISH/eval_robot.py +2025-01-06 17:28:48,207 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/lqn5ijm1-diff.patch +2025-01-06 17:28:48,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:48,233 INFO HandlerThread:2090663 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/tb" + save: true +} +control { + relay_id: "53cad69e1a284cc0844ac292d4426ea7" +} +_info { + stream_id: "d4l0r92a" +} + +2025-01-06 17:28:48,233 DEBUG HandlerThread:2090663 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:28:48,263 DEBUG SenderThread:2090663 [sender.py:send():379] send: telemetry +2025-01-06 17:28:48,284 DEBUG SenderThread:2090663 [sender.py:send():379] send: tbrecord +2025-01-06 17:28:48,285 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:28:48,285 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 with policy live +2025-01-06 17:28:48,471 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/dbm412tw-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:28:48,749 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/requirements.txt +2025-01-06 17:28:48,749 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-metadata.json +2025-01-06 17:28:48,749 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:28:48,749 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:28:48,750 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/diff.patch +2025-01-06 17:28:48,750 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code/FISH/eval_robot.py +2025-01-06 17:28:48,750 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code +2025-01-06 17:28:48,750 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code/FISH +2025-01-06 17:28:49,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:50,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:50,748 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:28:51,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:52,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:53,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:28:53,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:54,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:55,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:56,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:56,787 DEBUG SenderThread:2090663 [sender.py:send():379] send: config +2025-01-06 17:28:57,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:58,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:28:58,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:28:58,751 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:28:59,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:00,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:00,751 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:01,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:02,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:02,752 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:03,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:29:03,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:29:03,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:03,324 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:04,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:05,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:06,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:06,753 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:07,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:08,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:09,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:09,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:10,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:10,754 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:11,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:12,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:12,755 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:13,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:14,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:14,393 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:14,755 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:15,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:16,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:17,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:18,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:29:18,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:29:18,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:19,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:19,861 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:20,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:20,757 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml +2025-01-06 17:29:20,757 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:21,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:22,205 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:23,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:24,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:24,758 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:25,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:25,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:26,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:27,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:28,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:29,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:30,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:31,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:31,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:32,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:33,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:29:33,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:29:33,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:34,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:35,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:36,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:36,265 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:37,194 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:37,445 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:29:37,445 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 with policy now +2025-01-06 17:29:37,445 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:29:37,446 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:29:37,447 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:29:37,447 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:29:37,702 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/5j3gsj46-media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 +2025-01-06 17:29:37,761 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 +2025-01-06 17:29:37,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:29:37,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos +2025-01-06 17:29:37,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media +2025-01-06 17:29:37,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:29:38,016 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/tstaoxft-media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 +2025-01-06 17:29:38,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:38,762 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:39,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:40,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:41,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:41,597 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:42,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:42,763 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:43,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:44,195 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:44,764 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:29:45,196 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:46,196 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:47,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:47,306 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:47,846 DEBUG SystemMonitor:2090663 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:29:47,847 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:29:48,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:29:48,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:29:48,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:49,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:50,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:51,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:52,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:52,309 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:53,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:54,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:55,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:56,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:57,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:58,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:29:58,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:29:59,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:00,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:01,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:02,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:02,770 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:30:03,012 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/gcd6wjds-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:30:03,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:30:03,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:30:03,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:03,261 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:04,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:04,770 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:05,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:06,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:07,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:08,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:08,772 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:09,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:09,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:10,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:11,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:12,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:13,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:14,200 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:15,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:15,201 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:16,201 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:17,201 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:17,847 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:30:18,188 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:30:18,188 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:30:18,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:19,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:20,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:20,773 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:20,775 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:21,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:22,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:23,203 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:24,203 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:24,777 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:25,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:26,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:26,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:27,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:28,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:29,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:30,090 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:30:30,090 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 with policy now +2025-01-06 17:30:30,091 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:30:30,091 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:30:30,091 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:30:30,093 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:30:30,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:30,335 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/6qglpwso-media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 +2025-01-06 17:30:30,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:30:30,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:30,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 +2025-01-06 17:30:30,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:30:31,058 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/i569hmqq-media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 +2025-01-06 17:30:31,208 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:32,101 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:32,206 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:32,779 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:33,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:30:33,189 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:30:33,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:34,216 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:34,780 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:35,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:36,209 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:37,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:38,065 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:38,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:38,781 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:39,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:40,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:41,212 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:42,211 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:43,192 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:43,211 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:44,210 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:45,212 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:46,211 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:47,211 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:47,847 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:30:48,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:30:48,191 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:30:48,237 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:48,316 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:49,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:50,212 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:51,212 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:52,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:53,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:53,602 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:53,786 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:30:54,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:54,736 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:30:54,736 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:30:54,737 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:30:54,786 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:30:54,787 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:55,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:56,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:56,787 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:57,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:58,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:30:58,788 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:30:59,193 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:30:59,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:00,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:01,213 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:02,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:03,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:31:03,189 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:31:03,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:04,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:04,319 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:05,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:06,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:07,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:08,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:09,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:10,204 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:10,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:11,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:12,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:13,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:14,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:15,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:16,199 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:16,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:17,215 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:17,848 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:31:18,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:31:18,189 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:31:18,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:19,216 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:20,216 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:21,217 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:21,281 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:22,216 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:23,217 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:24,217 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:25,219 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:25,829 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:31:25,829 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 with policy now +2025-01-06 17:31:25,829 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:31:25,830 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:31:25,830 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:31:25,831 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:31:26,091 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/fwfcg65b-media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 +2025-01-06 17:31:26,223 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:26,797 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:31:26,797 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:26,798 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 +2025-01-06 17:31:26,798 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:31:26,851 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:27,096 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/gmh3byrr-media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 +2025-01-06 17:31:27,219 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:28,226 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:29,221 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:30,226 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:30,799 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:31,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:31,865 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:32,224 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:32,799 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:33,198 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:31:33,198 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:31:33,245 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:34,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:35,227 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:36,230 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:37,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:37,324 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:37,802 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:31:38,047 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/heuqkeu5-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:31:38,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:38,493 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:31:38,493 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:31:38,494 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:31:38,802 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:31:38,803 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:39,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:40,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:40,804 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:41,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:42,229 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:42,805 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:42,971 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:43,230 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:44,230 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:45,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:46,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:47,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:47,848 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:31:48,108 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:48,189 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:31:48,189 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:31:48,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:49,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:50,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:50,808 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:51,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:52,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:53,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:53,231 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:54,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:55,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:56,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:56,810 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:31:57,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:58,202 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:31:58,232 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:31:59,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:00,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:01,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:02,233 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:03,196 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:32:03,196 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:32:03,235 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:03,306 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:03,760 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:32:03,760 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 with policy now +2025-01-06 17:32:03,761 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:32:03,763 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:32:03,763 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:32:03,764 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:32:03,814 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:32:03,814 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 +2025-01-06 17:32:03,814 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:32:04,011 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/1buyzstt-media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 +2025-01-06 17:32:04,234 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:04,262 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/a4v633zo-media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 +2025-01-06 17:32:04,814 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:05,247 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:06,237 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:07,240 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:08,242 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:08,816 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:09,214 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:09,244 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:10,241 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:10,816 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:11,245 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:12,241 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:13,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:14,245 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:15,208 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:15,246 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:16,247 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:17,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:17,849 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:32:18,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:32:18,197 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:32:18,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:19,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:20,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:20,263 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:21,257 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:22,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:23,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:23,547 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:32:23,547 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:32:23,547 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:32:23,820 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:32:23,821 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:32:24,255 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:24,821 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:25,258 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:25,501 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:26,256 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:26,822 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:27,289 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:28,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:28,822 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:29,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:30,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:31,207 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:31,250 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:32,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:33,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:32:33,190 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:32:33,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:34,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:34,824 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:35,251 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:36,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:36,853 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:37,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:38,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:39,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:40,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:41,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:42,207 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:42,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:43,252 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:44,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:45,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:46,253 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:47,208 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:47,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:47,850 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:32:48,190 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:32:48,190 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:32:48,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:49,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:50,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:51,254 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:52,255 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:52,342 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:53,255 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:54,255 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:55,273 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:55,467 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:32:55,467 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 with policy now +2025-01-06 17:32:55,470 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:32:55,470 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:32:55,470 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:32:55,471 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:32:55,723 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/k6jof3hy-media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 +2025-01-06 17:32:55,831 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:32:55,832 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 +2025-01-06 17:32:55,832 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval +2025-01-06 17:32:56,096 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/2dsre9dk-media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 +2025-01-06 17:32:56,261 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:56,832 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:32:57,264 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:57,501 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:32:58,268 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:32:59,271 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:00,271 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:00,833 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:01,287 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:02,299 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:02,834 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:03,197 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:33:03,198 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:33:03,266 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:03,276 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:03,834 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:04,076 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/jzc7jh33-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:04,272 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:04,599 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:33:04,599 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:04,599 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:04,654 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:33:04,655 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval +2025-01-06 17:33:04,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table +2025-01-06 17:33:04,837 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:33:05,023 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/ev91wtea-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:33:05,120 INFO wandb-upload_2:2090663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplp0yrt7a +2025-01-06 17:33:05,181 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:33:05,191 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:33:05,272 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:05,594 INFO SenderThread:2090663 [sender.py:send_request_log_artifact():1525] logged artifact run-d4l0r92a-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjExMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMjQ2OQ==', 'latestArtifact': None}} +2025-01-06 17:33:05,594 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:33:05,594 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-06 17:33:05,595 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:33:05,595 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,596 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,597 DEBUG SenderThread:2090663 [sender.py:send():379] send: telemetry +2025-01-06 17:33:05,597 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,597 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,597 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,597 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,597 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,598 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,598 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,598 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,598 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,601 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,603 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,604 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,604 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,604 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,604 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,606 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,606 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,606 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,607 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,607 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,607 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:05,607 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:05,607 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:33:05,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:05,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:05,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-06 17:33:05,836 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval +2025-01-06 17:33:05,851 INFO wandb-upload_1:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/jwr53dwx-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:33:06,005 INFO wandb-upload_2:2090663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpylcyyb7s +2025-01-06 17:33:06,033 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/rz253b0p-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-06 17:33:06,272 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:06,389 INFO SenderThread:2090663 [sender.py:send_request_log_artifact():1525] logged artifact run-d4l0r92a-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjE0OQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMjQ3Mg==', 'latestArtifact': None}} +2025-01-06 17:33:06,389 DEBUG SenderThread:2090663 [sender.py:send():379] send: files +2025-01-06 17:33:06,390 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json with policy now +2025-01-06 17:33:06,390 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:33:06,390 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:06,391 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:06,638 INFO wandb-upload_1:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/x296pkve-media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-06 17:33:06,835 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:06,836 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:07,272 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:07,994 DEBUG SenderThread:2090663 [sender.py:send():379] send: exit +2025-01-06 17:33:07,994 INFO SenderThread:2090663 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:33:07,994 INFO SenderThread:2090663 [sender.py:send_exit():588] handling runtime: 260 +2025-01-06 17:33:07,994 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:07,994 INFO SenderThread:2090663 [sender.py:send_exit():594] send defer +2025-01-06 17:33:07,994 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:07,994 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:33:07,995 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:07,995 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:33:07,995 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:33:07,995 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:07,995 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:33:07,995 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:07,995 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:33:07,995 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:33:07,995 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:07,995 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:33:07,995 INFO HandlerThread:2090663 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:33:07,995 DEBUG SystemMonitor:2090663 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:33:07,995 DEBUG SystemMonitor:2090663 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:33:07,997 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:33:07,997 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:33:08,050 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:33:08,050 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:33:08,051 INFO HandlerThread:2090663 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:33:08,051 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:08,051 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:33:08,051 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:33:08,051 DEBUG SenderThread:2090663 [sender.py:send():379] send: stats +2025-01-06 17:33:08,051 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:08,051 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:33:08,052 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:08,052 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:33:08,052 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:33:08,052 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:08,052 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:33:08,836 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:08,837 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:09,837 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml +2025-01-06 17:33:14,604 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:14,605 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:14,605 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:33:14,605 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:14,606 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:14,606 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:33:14,606 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:33:14,606 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:14,606 DEBUG SenderThread:2090663 [sender.py:send():379] send: history +2025-01-06 17:33:14,606 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:33:14,606 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:14,607 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:33:14,607 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:14,607 DEBUG SenderThread:2090663 [sender.py:send():379] send: summary +2025-01-06 17:33:14,611 INFO SenderThread:2090663 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:33:14,611 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:14,611 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:33:14,611 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:33:14,611 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:14,611 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:33:14,612 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:14,612 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:33:14,612 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:33:14,612 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:14,612 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:14,612 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:33:14,612 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:14,612 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:33:14,839 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:14,995 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:17,998 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:33:17,998 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:17,998 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:17,998 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:33:17,999 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:17,999 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:33:17,999 INFO SenderThread:2090663 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:33:18,001 INFO SenderThread:2090663 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:33:18,003 INFO SenderThread:2090663 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:33:18,013 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:33:18,013 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:18,013 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:33:18,014 DEBUG SenderThread:2090663 [sender.py:send():379] send: artifact +2025-01-06 17:33:18,387 INFO wandb-upload_2:2090663 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp30y1vr2u +2025-01-06 17:33:18,388 INFO wandb-upload_1:2090663 [upload_job.py:push():85] Skipped uploading /tmp/tmpelzxd0ux/wandb-job.json +2025-01-06 17:33:18,481 INFO wandb-upload_0:2090663 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpf94pqpss +2025-01-06 17:33:18,840 INFO Thread-12 :2090663 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:18,943 INFO SenderThread:2090663 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkyMTY1Ng==', 'versionIndex': 423}}} +2025-01-06 17:33:18,943 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:18,943 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:33:18,943 INFO SenderThread:2090663 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:33:18,996 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:19,841 INFO SenderThread:2090663 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files +2025-01-06 17:33:19,841 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml config.yaml +2025-01-06 17:33:19,842 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json wandb-summary.json +2025-01-06 17:33:19,842 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/requirements.txt requirements.txt +2025-01-06 17:33:19,842 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:33:19,842 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:19,843 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log output.log +2025-01-06 17:33:19,844 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/diff.patch diff.patch +2025-01-06 17:33:19,849 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 media/videos/eval/3_eval_3_cac182b4027a7521827c.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 media/videos/eval/4_eval_4_12a3aaeba0fef5ed138d.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 media/videos/eval/0_eval_0_80d7712c707615309a21.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 media/videos/eval/2_eval_2_c6de2b4ebb693674e28c.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 media/videos/eval/1_eval_1_92182b2f19c7840dbbc4.mp4 +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json media/table/eval/total_success_rate_ci_6_3fbfe773b6a6adb9e9f8.table.json +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:33:19,850 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:33:19,850 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:19,851 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:19,851 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:33:19,851 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:19,853 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:19,853 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:33:19,854 INFO SenderThread:2090663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:33:19,996 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:19,997 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:20,033 INFO wandb-upload_2:2090663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/config.yaml +2025-01-06 17:33:20,036 INFO wandb-upload_0:2090663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/wandb-summary.json +2025-01-06 17:33:20,088 INFO wandb-upload_1:2090663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/requirements.txt +2025-01-06 17:33:20,109 INFO wandb-upload_3:2090663 [upload_job.py:push():130] Uploaded file /tmp/tmp6hnk1gbfwandb/sot7vj7s-events.out.tfevents.1736202528.leonmkim-ROG-Strix-G15CS-G15CS.2090492.0 +2025-01-06 17:33:20,117 INFO wandb-upload_4:2090663 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/files/output.log +2025-01-06 17:33:20,318 INFO Thread-11 (_thread_body):2090663 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:33:20,318 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:20,318 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:33:20,318 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:20,318 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:33:20,318 INFO SenderThread:2090663 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:33:20,319 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:33:20,319 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:20,320 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:33:20,320 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:20,320 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:33:20,320 INFO SenderThread:2090663 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:33:20,390 INFO SenderThread:2090663 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:33:20,390 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:33:20,390 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:20,391 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:33:20,391 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:20,391 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:33:20,391 INFO SenderThread:2090663 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:33:20,391 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:33:20,391 INFO HandlerThread:2090663 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:33:20,391 DEBUG SenderThread:2090663 [sender.py:send():379] send: final +2025-01-06 17:33:20,391 DEBUG SenderThread:2090663 [sender.py:send():379] send: footer +2025-01-06 17:33:20,391 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: defer +2025-01-06 17:33:20,391 INFO SenderThread:2090663 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:33:20,392 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:20,392 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:33:20,392 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:33:20,392 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:33:20,393 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:20,393 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:33:20,393 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:33:20,395 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:20,453 DEBUG SenderThread:2090663 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:33:20,454 INFO MainThread:2090663 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:33:20,454 INFO MainThread:2090663 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:33:20,455 INFO MainThread:2090663 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:33:20,455 DEBUG HandlerThread:2090663 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:33:20,455 INFO HandlerThread:2090663 [handler.py:finish():882] shutting down handler +2025-01-06 17:33:21,393 INFO WriterThread:2090663 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/run-d4l0r92a.wandb +2025-01-06 17:33:21,453 INFO SenderThread:2090663 [sender.py:finish():1615] shutting down sender +2025-01-06 17:33:21,454 INFO SenderThread:2090663 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:33:21,454 INFO SenderThread:2090663 [file_pusher.py:join():175] waiting for file pusher diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug.log b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..517f79c4d32ee10b3e15d8d9ce2731f0ec1040c3 --- /dev/null +++ b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Configure stats pid to 2090492 +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/settings +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:28:47,449 INFO MainThread:2090492 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug.log +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/logs/debug-internal.log +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a'} +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:init():619] starting backend +2025-01-06 17:28:47,450 INFO MainThread:2090492 [wandb_init.py:init():623] setting up manager +2025-01-06 17:28:47,454 INFO MainThread:2090492 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:28:47,455 INFO MainThread:2090492 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:28:47,468 INFO MainThread:2090492 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:28:47,475 INFO MainThread:2090492 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:28:47,762 INFO MainThread:2090492 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:28:47,833 INFO MainThread:2090492 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:28:47,833 INFO MainThread:2090492 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:28:48,188 INFO MainThread:2090492 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:28:48,188 INFO MainThread:2090492 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:28:48,188 INFO MainThread:2090492 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:28:48,188 INFO MainThread:2090492 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:28:48,190 INFO MainThread:2090492 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:28:48,190 INFO MainThread:2090492 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/d4l0r92a/tb, True +2025-01-06 17:28:56,786 INFO MainThread:2090492 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:33:21,455 WARNING MsgRouterThr:2090492 [router.py:message_loop():77] message_loop has been closed diff --git a/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/run-d4l0r92a.wandb b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/run-d4l0r92a.wandb new file mode 100644 index 0000000000000000000000000000000000000000..41a7b79299d75bd34cb08a89d6efdde465b2d84b Binary files /dev/null and b/d4l0r92a/wandb/run-20250106_172847-d4l0r92a/run-d4l0r92a.wandb differ diff --git a/e3yh61zm/episode_rosbags/episode_0_2025-01-07-17-55-52.bag b/e3yh61zm/episode_rosbags/episode_0_2025-01-07-17-55-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..1cb009a1ca1a9e75638095cf8aeba6def5b092ad --- /dev/null +++ b/e3yh61zm/episode_rosbags/episode_0_2025-01-07-17-55-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc4ee1b4d96d489d8ce757790e17b57c6236c093f358740563e6951d5f458ed4 +size 1406304081 diff --git a/e3yh61zm/episode_rosbags/episode_3_2025-01-07-17-58-09.bag b/e3yh61zm/episode_rosbags/episode_3_2025-01-07-17-58-09.bag new file mode 100644 index 0000000000000000000000000000000000000000..e93c7781c3240fd5ac6438151d2b37769ebd8116 --- /dev/null +++ b/e3yh61zm/episode_rosbags/episode_3_2025-01-07-17-58-09.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac9e74c2a4d95600aa916d5c6c5acbc64628cd12758e92d0be6bac351be7fb70 +size 1411894100 diff --git a/e3yh61zm/episode_rosbags/episode_4_2025-01-07-17-58-51.bag b/e3yh61zm/episode_rosbags/episode_4_2025-01-07-17-58-51.bag new file mode 100644 index 0000000000000000000000000000000000000000..eb18da4449fc823449fdbde4e520110b903defbe --- /dev/null +++ b/e3yh61zm/episode_rosbags/episode_4_2025-01-07-17-58-51.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:538454beccc84a61bbbc269c43c5198f65407b8aa8c85fa045a545bddd5da9bc +size 1408170469 diff --git a/i3pbiz9p/episode_rosbags/episode_1_2025-01-06-17-34-41.bag b/i3pbiz9p/episode_rosbags/episode_1_2025-01-06-17-34-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..623fc2d56043f55556370bcace3796271a6ea098 --- /dev/null +++ b/i3pbiz9p/episode_rosbags/episode_1_2025-01-06-17-34-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:326fad48d7c2281b140ab0d2b4548b1f684276db8a55017d1b8316aa3cc0a584 +size 1412672341 diff --git a/jwgj4azt/.hydra/config.yaml b/jwgj4azt/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/jwgj4azt/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/jwgj4azt/.hydra/hydra.yaml b/jwgj4azt/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1691a76c0eaf1d921fe66a752323abbbef694456 --- /dev/null +++ b/jwgj4azt/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/jwgj4azt/.hydra/overrides.yaml b/jwgj4azt/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/jwgj4azt/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/jwgj4azt/episode_rosbags/aligned_depth_to_color_K.npy b/jwgj4azt/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/jwgj4azt/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/jwgj4azt/episode_rosbags/cam_tf_world.npy b/jwgj4azt/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/jwgj4azt/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/jwgj4azt/episode_rosbags/color_K.npy b/jwgj4azt/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/jwgj4azt/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/jwgj4azt/episode_rosbags/depth_K.npy b/jwgj4azt/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/jwgj4azt/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/jwgj4azt/episode_rosbags/episode_0_2025-01-06-20-32-00.bag b/jwgj4azt/episode_rosbags/episode_0_2025-01-06-20-32-00.bag new file mode 100644 index 0000000000000000000000000000000000000000..0ac9ce5302944146c47205c031e95052e6453e5a --- /dev/null +++ b/jwgj4azt/episode_rosbags/episode_0_2025-01-06-20-32-00.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1adc26c40d8dbf274b8e2669727b7d5c51005f746cc1ff36fe286573dbb54b99 +size 1412884741 diff --git a/jwgj4azt/episode_rosbags/episode_1_2025-01-06-20-32-39.bag b/jwgj4azt/episode_rosbags/episode_1_2025-01-06-20-32-39.bag new file mode 100644 index 0000000000000000000000000000000000000000..2ba034d4276005c20f1e9a0b143d7f99ab0c8508 --- /dev/null +++ b/jwgj4azt/episode_rosbags/episode_1_2025-01-06-20-32-39.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02c6a97eaf82a73d078de8afb240f7f9850037d6171bc4a8473b1f2d8745aad4 +size 1401850339 diff --git a/jwgj4azt/episode_rosbags/episode_2_2025-01-06-20-33-19.bag b/jwgj4azt/episode_rosbags/episode_2_2025-01-06-20-33-19.bag new file mode 100644 index 0000000000000000000000000000000000000000..b1f44c78f0654bd80493136ade4a906d12f29c23 --- /dev/null +++ b/jwgj4azt/episode_rosbags/episode_2_2025-01-06-20-33-19.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb6ab5b2f0379a3b33bc2f7abd397ff6a5b95016195e30788b3061f88783e78d +size 1411750339 diff --git a/jwgj4azt/episode_rosbags/episode_3_2025-01-06-20-33-59.bag b/jwgj4azt/episode_rosbags/episode_3_2025-01-06-20-33-59.bag new file mode 100644 index 0000000000000000000000000000000000000000..a961808d088afeae2ad8fbe510f39b07b7ca3a3b --- /dev/null +++ b/jwgj4azt/episode_rosbags/episode_3_2025-01-06-20-33-59.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:27ba4357e72870b3d99e20a9aefe9f4d76288d4ed8c0a529885cdcd67079fb0e +size 1411333074 diff --git a/jwgj4azt/episode_rosbags/episode_4_2025-01-06-20-34-40.bag b/jwgj4azt/episode_rosbags/episode_4_2025-01-06-20-34-40.bag new file mode 100644 index 0000000000000000000000000000000000000000..93d310b933fdf1599fe2a966bc9f8d64821d39c0 --- /dev/null +++ b/jwgj4azt/episode_rosbags/episode_4_2025-01-06-20-34-40.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:03ff4c7b867a08c4ff761ec1480e1406c98ba399627d257d7360f9d012aa4622 +size 1407786899 diff --git a/jwgj4azt/eval_robot.log b/jwgj4azt/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..5ecca6702181b3c09dfb65dc049effa35414428b --- /dev/null +++ b/jwgj4azt/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 20:31:25,471][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:31:25,473][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/jwgj4azt/eval_video/0_eval.mp4 b/jwgj4azt/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f38a75bd6203acec3128d874a0a6cb9531a7b1f6 --- /dev/null +++ b/jwgj4azt/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c19b9ddad84cf3e30386668b1ce4f00bca8b34974ab73f345354a77fd2484162 +size 1143520 diff --git a/jwgj4azt/eval_video/1_eval.mp4 b/jwgj4azt/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bf83160ca0eb908e40832443b23e310ed5acfd50 --- /dev/null +++ b/jwgj4azt/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e671e1fc9c3432634515aa8a65c79d3612fb9df1d6b9e5ee0a85c2d2eededc6 +size 1149197 diff --git a/jwgj4azt/eval_video/2_eval.mp4 b/jwgj4azt/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..efbec11e03ed08291e0f42bc57b60f3e17f914e6 --- /dev/null +++ b/jwgj4azt/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da772e21c938b47e3f0aee289e1b2029e73f393a2ce93590b0859f47887b948 +size 1150379 diff --git a/jwgj4azt/eval_video/3_eval.mp4 b/jwgj4azt/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b39a9743d49d12481af18a9dac7fb12b00287786 --- /dev/null +++ b/jwgj4azt/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4c60367d0b4d6cdfda4ef568547dad9d0fe06f74a235ceea5cc5cb024bb956c +size 1149174 diff --git a/jwgj4azt/eval_video/4_eval.mp4 b/jwgj4azt/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a308bcd9c5a8ef6042cf662056a7d7cda2a5d30 --- /dev/null +++ b/jwgj4azt/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24ed8d7ed14931d68e7d126977166aacc6caf913f3829e6c967dd00e1822f1d8 +size 1149294 diff --git a/jwgj4azt/tb/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 b/jwgj4azt/tb/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 new file mode 100644 index 0000000000000000000000000000000000000000..2027ac0e74ca620ba988f0052fe75934d58cd146 --- /dev/null +++ b/jwgj4azt/tb/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e9088feb993c210f7ca8c8f5b9feecf96c40afd3476b3f915e92802234612a9 +size 6539 diff --git a/jwgj4azt/wandb/debug-internal.log b/jwgj4azt/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..255e791a29cf971a8573193d8ddd483ccbd831c6 --- /dev/null +++ b/jwgj4azt/wandb/debug-internal.log @@ -0,0 +1,732 @@ +2025-01-06 20:31:31,769 INFO StreamThr :2110981 [internal.py:wandb_internal():85] W&B internal server running at pid: 2110981, started at: 2025-01-06 20:31:31.769206 +2025-01-06 20:31:31,770 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:31:31,782 INFO WriterThread:2110981 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/run-jwgj4azt.wandb +2025-01-06 20:31:31,782 DEBUG SenderThread:2110981 [sender.py:send():379] send: header +2025-01-06 20:31:31,792 DEBUG SenderThread:2110981 [sender.py:send():379] send: run +2025-01-06 20:31:32,024 INFO SenderThread:2110981 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files +2025-01-06 20:31:32,024 INFO SenderThread:2110981 [sender.py:_start_run_threads():1188] run started: jwgj4azt with start time 1736213491.769888 +2025-01-06 20:31:32,037 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:31:32,038 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:31:32,096 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:31:32,101 DEBUG HandlerThread:2110981 [system_info.py:__init__():26] System info init +2025-01-06 20:31:32,101 DEBUG HandlerThread:2110981 [system_info.py:__init__():41] System info init done +2025-01-06 20:31:32,101 INFO HandlerThread:2110981 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:31:32,101 INFO SystemMonitor:2110981 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:31:32,101 INFO HandlerThread:2110981 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:31:32,102 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:31:32,102 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:31:32,103 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:31:32,103 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:31:32,103 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:31:32,115 DEBUG HandlerThread:2110981 [system_info.py:probe():152] Probing system +2025-01-06 20:31:32,117 DEBUG HandlerThread:2110981 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:31:32,122 DEBUG HandlerThread:2110981 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:31:32,122 DEBUG HandlerThread:2110981 [system_info.py:probe():200] Probing system done +2025-01-06 20:31:32,122 DEBUG HandlerThread:2110981 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:31:32.116005', 'startedAt': '2025-01-07T01:31:31.760526', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4499.796625000001, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.014, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 4500.022, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 498.98254013061523}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:31:32,122 INFO HandlerThread:2110981 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:31:32,122 INFO HandlerThread:2110981 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:31:32,122 DEBUG HandlerThread:2110981 [system_info.py:_save_code():44] Saving code +2025-01-06 20:31:32,124 DEBUG HandlerThread:2110981 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:31:32,124 DEBUG HandlerThread:2110981 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:31:32,186 DEBUG HandlerThread:2110981 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:31:32,187 INFO HandlerThread:2110981 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:31:32,190 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:31:32,190 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:31:32,190 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:31:32,190 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:31:32,439 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/3d21j0cb-wandb-metadata.json +2025-01-06 20:31:32,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:31:32,444 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:31:32,445 DEBUG SenderThread:2110981 [sender.py:send():379] send: telemetry +2025-01-06 20:31:32,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:32,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:31:32,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:31:32,448 INFO HandlerThread:2110981 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/tb" + save: true +} +control { + relay_id: "7ce92a542acc48e38ea9e8d4834a33f7" +} +_info { + stream_id: "jwgj4azt" +} + +2025-01-06 20:31:32,449 DEBUG HandlerThread:2110981 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:31:32,475 INFO wandb-upload_1:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/5odjnous-code/FISH/eval_robot.py +2025-01-06 20:31:32,515 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/y6oe653p-diff.patch +2025-01-06 20:31:32,539 DEBUG SenderThread:2110981 [sender.py:send():379] send: tbrecord +2025-01-06 20:31:32,539 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:31:32,539 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 with policy live +2025-01-06 20:31:32,734 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/i5kuvii2-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:31:33,026 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-metadata.json +2025-01-06 20:31:33,026 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/diff.patch +2025-01-06 20:31:33,026 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code/FISH/eval_robot.py +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/requirements.txt +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code/FISH +2025-01-06 20:31:33,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:34,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:35,025 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:35,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:36,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:37,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:37,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:38,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:39,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:40,165 DEBUG SenderThread:2110981 [sender.py:send():379] send: config +2025-01-06 20:31:40,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:41,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:42,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:42,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:43,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:43,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:44,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:45,028 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:45,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:46,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:47,029 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:47,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:31:47,444 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:31:47,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:47,536 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:48,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:49,029 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:49,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:50,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:51,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:52,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:53,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:53,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:54,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:55,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:56,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:57,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:58,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:58,693 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:59,032 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:59,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:00,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:01,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:02,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:32:02,444 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:32:02,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:03,033 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:03,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:04,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:04,536 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:05,034 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml +2025-01-06 20:32:05,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:06,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:07,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:08,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:09,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:10,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:10,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:11,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:12,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:13,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:14,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:15,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:15,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:16,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:17,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:32:17,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:32:17,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:18,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:19,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:20,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:20,551 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:21,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:22,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:23,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:24,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:25,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:26,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:26,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:27,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:28,325 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:32:28,325 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 with policy now +2025-01-06 20:32:28,325 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:32:28,326 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:32:28,326 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:32:28,327 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:32:28,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:28,749 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/jfz6y511-media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 +2025-01-06 20:32:29,040 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos +2025-01-06 20:32:29,316 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/l0of5mt0-media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 +2025-01-06 20:32:29,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:30,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:31,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:31,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:32,033 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:32,103 DEBUG SystemMonitor:2110981 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:32:32,104 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:32:32,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:32:32,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:32:32,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:33,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:33,454 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:34,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:35,454 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:36,042 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:32:36,301 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/fn7ymk3h-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:32:36,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:37,043 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:37,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:37,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:38,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:39,044 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:39,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:40,454 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:41,044 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:41,459 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:42,458 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:43,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:43,458 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:44,458 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:45,459 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:46,459 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:47,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:32:47,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:32:47,489 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:48,459 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:48,532 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:49,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:50,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:51,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:52,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:53,048 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:53,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:53,765 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:54,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:55,049 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:55,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:56,464 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:57,463 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:58,463 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:59,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:59,464 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:00,464 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:01,464 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:02,105 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:33:02,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:33:02,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:33:02,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:03,465 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:04,465 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:04,547 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:05,472 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:06,471 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:07,493 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:07,697 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:33:07,698 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:33:07,698 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 with policy now +2025-01-06 20:33:07,698 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:33:07,698 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:33:07,699 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:33:07,969 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/nj9q2ipb-media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 +2025-01-06 20:33:08,053 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:33:08,053 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 +2025-01-06 20:33:08,053 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:33:08,312 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/5et240k5-media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 +2025-01-06 20:33:08,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:09,053 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:09,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:09,700 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:10,474 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:11,473 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:12,479 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:13,054 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:13,479 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:14,474 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:15,321 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:15,474 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:16,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:16,863 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:33:16,863 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:33:16,870 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:33:17,056 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:17,056 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:33:17,056 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:33:17,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:33:17,449 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:33:17,479 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:18,484 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:19,056 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:19,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:20,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:21,057 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:21,275 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:21,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:22,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:23,058 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:23,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:24,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:25,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:26,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:26,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:27,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:28,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:29,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:30,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:31,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:31,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:32,105 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:33:32,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:33:32,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:33:32,489 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:33,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:34,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:35,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:36,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:36,607 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:37,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:38,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:39,063 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:39,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:40,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:41,064 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:41,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:41,857 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:42,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:43,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:44,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:45,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:46,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:46,988 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:47,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:33:47,447 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:33:47,481 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:47,979 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:33:47,979 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 with policy now +2025-01-06 20:33:47,981 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:33:47,982 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:33:47,982 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:33:47,983 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:33:48,066 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:33:48,066 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 +2025-01-06 20:33:48,066 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:33:48,268 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/s5sq8j85-media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 +2025-01-06 20:33:48,481 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:48,528 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/50aovh5p-media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 +2025-01-06 20:33:49,066 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:49,484 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:50,483 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:51,483 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:52,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:52,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:53,068 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:53,483 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:54,616 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:55,613 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:56,069 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:33:56,313 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/3wmnsj9n-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:33:56,615 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:56,694 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:33:56,694 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:33:56,694 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:33:57,069 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:57,070 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:33:57,454 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:57,614 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:58,618 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:59,070 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:59,629 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:00,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:01,071 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:01,618 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:02,106 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:34:02,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:34:02,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:34:02,576 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:02,618 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:03,071 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:03,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:04,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:05,072 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:05,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:06,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:07,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:08,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:08,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:09,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:10,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:11,074 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:11,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:12,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:13,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:14,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:14,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:15,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:16,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:17,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:34:17,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:34:17,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:18,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:19,594 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:19,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:20,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:21,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:22,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:23,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:24,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:25,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:25,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:26,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:27,627 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:27,845 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:34:27,845 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:34:27,846 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 with policy now +2025-01-06 20:34:27,846 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:34:27,846 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:34:27,847 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:34:28,080 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:34:28,080 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 +2025-01-06 20:34:28,080 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:34:28,119 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/d98zxpk2-media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 +2025-01-06 20:34:28,413 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/le7fj6rq-media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 +2025-01-06 20:34:28,634 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:29,080 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:29,641 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:30,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:30,860 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:31,640 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:32,106 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:34:32,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:34:32,449 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:34:32,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:33,081 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:33,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:34,640 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:35,641 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:36,163 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:36,647 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:36,743 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:34:36,743 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:34:36,743 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:34:37,083 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:37,083 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:34:37,083 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:34:37,649 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:38,649 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:39,084 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:39,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:40,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:41,084 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:41,466 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:41,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:42,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:43,085 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:43,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:44,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:45,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:46,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:47,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:34:47,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:34:47,537 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:47,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:48,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:49,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:50,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:51,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:52,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:53,088 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:53,457 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:53,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:54,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:55,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:56,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:57,647 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:58,457 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:58,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:59,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:00,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:01,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:02,106 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:35:02,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:35:02,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:35:02,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:03,568 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:03,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:04,646 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:05,646 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:06,646 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:07,653 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:08,215 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:35:08,215 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 with policy now +2025-01-06 20:35:08,219 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:35:08,220 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:08,220 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:08,221 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:08,486 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/mhwvxn89-media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 +2025-01-06 20:35:08,650 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:09,094 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:09,094 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:09,094 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 +2025-01-06 20:35:09,095 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:35:09,241 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:09,397 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/f9ubcnjy-media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 +2025-01-06 20:35:09,649 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:10,655 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:11,656 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:12,667 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:13,095 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:13,659 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:14,470 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:14,664 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:15,655 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:16,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:17,097 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:17,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:35:17,451 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:35:17,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:18,665 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:19,597 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:19,667 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:20,665 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:20,796 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:20,796 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:20,797 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:20,851 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:35:20,851 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:35:21,024 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:35:21,098 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:21,098 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:21,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:35:21,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media +2025-01-06 20:35:21,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval +2025-01-06 20:35:21,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table +2025-01-06 20:35:21,291 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/kuj5jjbj-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:35:21,347 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/ksk4cpuz-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:21,362 INFO wandb-upload_1:2110981 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0q7m3kfb +2025-01-06 20:35:21,420 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:35:21,429 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:35:21,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:22,030 INFO SenderThread:2110981 [sender.py:send_request_log_artifact():1525] logged artifact run-jwgj4azt-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI5MDE0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYzMjAyMQ==', 'latestArtifact': None}} +2025-01-06 20:35:22,030 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:35:22,030 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:35:22,031 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:22,031 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,031 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,031 DEBUG SenderThread:2110981 [sender.py:send():379] send: telemetry +2025-01-06 20:35:22,031 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,035 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,035 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,036 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,036 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,036 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,036 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,037 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,037 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,037 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,037 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,038 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,038 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,038 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,038 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,038 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,039 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,039 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,039 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,039 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,040 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,040 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,040 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:35:22,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:22,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:22,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json +2025-01-06 20:35:22,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval +2025-01-06 20:35:22,289 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/d881s9ye-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:35:22,293 INFO wandb-upload_1:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/ey9moiko-media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json +2025-01-06 20:35:22,435 INFO wandb-upload_2:2110981 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4ks2i1aa +2025-01-06 20:35:22,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:22,898 INFO SenderThread:2110981 [sender.py:send_request_log_artifact():1525] logged artifact run-jwgj4azt-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI5MDIzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYzMjEwMQ==', 'latestArtifact': None}} +2025-01-06 20:35:22,898 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:35:22,898 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json with policy now +2025-01-06 20:35:22,898 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:22,898 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,899 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:23,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:23,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:23,131 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/xozmhro4-media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json +2025-01-06 20:35:23,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:24,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:24,899 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:25,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:26,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:27,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:28,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:29,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:30,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:30,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:31,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:32,107 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:35:32,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:35:32,447 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:35:32,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:32,925 DEBUG SenderThread:2110981 [sender.py:send():379] send: exit +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:send_exit():588] handling runtime: 240 +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:send_exit():594] send defer +2025-01-06 20:35:32,926 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,926 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:35:32,926 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:35:32,927 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,927 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:35:32,927 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:32,927 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:35:32,927 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:35:32,927 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,927 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:35:32,927 INFO HandlerThread:2110981 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:35:32,927 DEBUG SystemMonitor:2110981 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:35:32,927 DEBUG SystemMonitor:2110981 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:35:32,928 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:35:32,928 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:35:32,979 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:35:32,979 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:35:32,979 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:35:32,980 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:32,980 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:35:32,980 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:35:32,980 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:35:32,980 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,980 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:35:32,980 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:32,980 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:35:32,980 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:35:32,980 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,980 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:35:33,103 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:35,104 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:39,802 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:39,802 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:39,802 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:35:39,803 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:39,803 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:35:39,803 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:35:39,803 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:39,804 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:39,804 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:39,804 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:39,804 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:35:39,804 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:39,805 DEBUG SenderThread:2110981 [sender.py:send():379] send: summary +2025-01-06 20:35:39,805 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:39,805 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:39,805 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:35:39,805 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:35:39,805 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:39,806 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:35:39,806 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:39,806 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:35:39,809 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:39,926 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:35:39,927 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:39,927 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:35:39,927 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:39,927 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:35:39,927 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:40,106 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:40,106 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml +2025-01-06 20:35:40,789 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:35:40,789 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:40,789 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:40,789 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:35:40,790 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:40,790 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:35:40,790 INFO SenderThread:2110981 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:35:40,792 INFO SenderThread:2110981 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:35:40,794 INFO SenderThread:2110981 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:35:40,811 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:35:40,811 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:40,811 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:35:40,811 DEBUG SenderThread:2110981 [sender.py:send():379] send: artifact +2025-01-06 20:35:40,928 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:41,013 INFO SenderThread:2110981 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:35:41,013 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,014 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:35:41,014 INFO SenderThread:2110981 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:35:41,106 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml config.yaml +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json wandb-summary.json +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/requirements.txt requirements.txt +2025-01-06 20:35:41,108 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:35:41,108 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log output.log +2025-01-06 20:35:41,108 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/diff.patch diff.patch +2025-01-06 20:35:41,110 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:35:41,112 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json +2025-01-06 20:35:41,117 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:35:41,117 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:35:41,117 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:41,117 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,117 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:35:41,117 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,117 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:35:41,117 INFO SenderThread:2110981 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:35:41,288 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml +2025-01-06 20:35:41,298 INFO wandb-upload_1:2110981 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:41,362 INFO wandb-upload_4:2110981 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/requirements.txt +2025-01-06 20:35:41,376 INFO wandb-upload_3:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/7gebbo7f-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:41,379 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:41,580 INFO Thread-11 (_thread_body):2110981 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:35:41,580 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,580 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:35:41,580 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,580 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:35:41,580 INFO SenderThread:2110981 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:35:41,581 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:35:41,582 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,582 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:35:41,582 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,582 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:35:41,582 INFO SenderThread:2110981 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:35:41,805 INFO SenderThread:2110981 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:35:41,805 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:35:41,805 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,805 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:35:41,806 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,806 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:35:41,806 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:35:41,806 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,806 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:35:41,806 DEBUG SenderThread:2110981 [sender.py:send():379] send: final +2025-01-06 20:35:41,806 DEBUG SenderThread:2110981 [sender.py:send():379] send: footer +2025-01-06 20:35:41,806 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,806 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:35:41,807 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:41,807 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:41,807 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:35:41,807 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:41,808 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:35:41,808 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:35:41,808 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:35:41,810 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:41,860 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:41,860 INFO MainThread:2110981 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:35:41,860 INFO MainThread:2110981 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:35:41,860 INFO MainThread:2110981 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:35:41,861 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:35:41,861 INFO HandlerThread:2110981 [handler.py:finish():882] shutting down handler +2025-01-06 20:35:42,808 INFO WriterThread:2110981 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/run-jwgj4azt.wandb +2025-01-06 20:35:42,860 INFO SenderThread:2110981 [sender.py:finish():1615] shutting down sender +2025-01-06 20:35:42,860 INFO SenderThread:2110981 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:35:42,860 INFO SenderThread:2110981 [file_pusher.py:join():175] waiting for file pusher diff --git a/jwgj4azt/wandb/debug.log b/jwgj4azt/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d3ff7a1424750ac620034ef2043a8372bd23031b --- /dev/null +++ b/jwgj4azt/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Configure stats pid to 2110680 +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/settings +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug.log +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug-internal.log +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:31:31,763 INFO MainThread:2110680 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt'} +2025-01-06 20:31:31,763 INFO MainThread:2110680 [wandb_init.py:init():619] starting backend +2025-01-06 20:31:31,763 INFO MainThread:2110680 [wandb_init.py:init():623] setting up manager +2025-01-06 20:31:31,768 INFO MainThread:2110680 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:31:31,769 INFO MainThread:2110680 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:31:31,781 INFO MainThread:2110680 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:31:31,790 INFO MainThread:2110680 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:31:32,037 INFO MainThread:2110680 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:31:32,091 INFO MainThread:2110680 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:31:32,091 INFO MainThread:2110680 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:31:32,444 INFO MainThread:2110680 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:31:32,444 INFO MainThread:2110680 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:31:32,444 INFO MainThread:2110680 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:31:32,444 INFO MainThread:2110680 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:31:32,446 INFO MainThread:2110680 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:31:32,446 INFO MainThread:2110680 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/tb, True +2025-01-06 20:31:40,164 INFO MainThread:2110680 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:35:42,861 WARNING MsgRouterThr:2110680 [router.py:message_loop():77] message_loop has been closed diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code/FISH/eval_robot.py b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..71f99256342b9a3f78fa3ca62644690ae563c2d7 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736213491 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: fowlers +left_book_slot: + desc: null + value: twodim diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/diff.patch b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c28171b0ccbf1f8e5d0d27b96acf339a03de1c06 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..a719175 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0b5968f06249e38f350c647ccab1bdf656a458d6 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[5, 15, 0.3333333333333333, 0.10265967257817107, 0.6413327455237964, 0.14299744374739537, 0.6138283610116277, 0.1682697982259454, 0.5397089421669697, 0.17985042190994288, 0.5254023298570373]]} \ No newline at end of file diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f38a75bd6203acec3128d874a0a6cb9531a7b1f6 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c19b9ddad84cf3e30386668b1ce4f00bca8b34974ab73f345354a77fd2484162 +size 1143520 diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bf83160ca0eb908e40832443b23e310ed5acfd50 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e671e1fc9c3432634515aa8a65c79d3612fb9df1d6b9e5ee0a85c2d2eededc6 +size 1149197 diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..efbec11e03ed08291e0f42bc57b60f3e17f914e6 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4da772e21c938b47e3f0aee289e1b2029e73f393a2ce93590b0859f47887b948 +size 1150379 diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b39a9743d49d12481af18a9dac7fb12b00287786 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4c60367d0b4d6cdfda4ef568547dad9d0fe06f74a235ceea5cc5cb024bb956c +size 1149174 diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..9a308bcd9c5a8ef6042cf662056a7d7cda2a5d30 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24ed8d7ed14931d68e7d126977166aacc6caf913f3829e6c967dd00e1822f1d8 +size 1149294 diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a785cd13223ce8310e082b8fe1ade5b3b17849e4 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log @@ -0,0 +1,268 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736213500.165238]: resetting environment +[INFO] [1736213500.168069]: cleared current plan +[INFO] [1736213500.169086]: moving to home +[INFO] [1736213501.171621]: reached home +[INFO] [1736213501.172939]: reset action history +[INFO] [1736213503.883251]: environment reset +Starting episode 0 +[INFO] [1736213503.885611]: resetting environment +[INFO] [1736213503.891794]: cleared current plan +[INFO] [1736213503.892737]: moving to home +[INFO] [1736213504.894863]: reached home +[INFO] [1736213504.896378]: reset action history +[INFO] [1736213507.604670]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736213516.644670]: resetting environment +[INFO] [1736213516.648556]: cleared current plan +[INFO] [1736213516.649366]: moving to home +[INFO] [1736213517.650996]: reached home +[INFO] [1736213517.652450]: reset action history +[INFO] [1736213520.360368]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/episode_rosbags/episode_0_2025-01-06-20-32-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2110680.log +started roslaunch server http://158.130.50.37:36509/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2111099] +process[print_text-2]: started with pid [2111123] +process[rosbag_record-3]: started with pid [2111124] +started bagging! +[INFO] [1736213547.370668]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736213548.116628]: Stored episode 1. +[INFO] [1736213548.117376]: Saving video... +[INFO] [1736213548.315581]: Video saved! +Starting episode 1 +[INFO] [1736213548.326663]: resetting environment +[INFO] [1736213548.330931]: cleared current plan +[INFO] [1736213548.331130]: moving to home +[INFO] [1736213552.033281]: reached home +[INFO] [1736213552.034500]: reset action history +[INFO] [1736213554.741648]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213555.884000]: resetting environment +[INFO] [1736213555.889866]: cleared current plan +[INFO] [1736213555.890048]: moving to home +[INFO] [1736213556.891510]: reached home +[INFO] [1736213556.892375]: reset action history +[INFO] [1736213559.600277]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/episode_rosbags/episode_1_2025-01-06-20-32-39.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2110680.log +started roslaunch server http://158.130.50.37:35955/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2111261] +process[print_text-5]: started with pid [2111262] +process[rosbag_record-6]: started with pid [2111263] +started bagging! +For topic EE_pose: timestamp difference is 172307545 for nearest: 1736213570711242014 - target: 1736213570538934469 at idx 0 +For topic gripper_width: timestamp difference is -109228582 for nearest: 1736213572533268481 - target: 1736213572642497063 at idx 185 +For topic gripper_width: timestamp difference is -106333333 for nearest: 1736213573933255357 - target: 1736213574039588690 at idx 203 +[INFO] [1736213586.692066]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit[print_text-5] killing on exit +[INFO] [1736213587.450833]: Stored episode 2. +[INFO] [1736213587.453153]: Saving video... +[INFO] [1736213587.650259]: Video saved! +Starting episode 2 +[INFO] [1736213587.678319]: resetting environment +[INFO] [1736213587.696623]: cleared current plan +[INFO] [1736213587.697627]: moving to home +[INFO] [1736213591.503016]: reached home +[INFO] [1736213591.504611]: reset action history +[INFO] [1736213594.277899]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213596.091752]: resetting environment +[INFO] [1736213596.112203]: cleared current plan +[INFO] [1736213596.114836]: moving to home +[INFO] [1736213597.118447]: reached home +[INFO] [1736213597.119443]: reset action history +[INFO] [1736213599.899328]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/episode_rosbags/episode_2_2025-01-06-20-33-19.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2110680.log +started roslaunch server http://158.130.50.37:44733/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2111395] +process[print_text-8]: started with pid [2111396] +process[rosbag_record-9]: started with pid [2111397] +started bagging! +For topic gripper_width: timestamp difference is 103048699 for nearest: 1736213616399901526 - target: 1736213616296852827 at idx 236 +For topic gripper_width: timestamp difference is 103267602 for nearest: 1736213616599925689 - target: 1736213616496658087 at idx 238 +For topic gripper_width: timestamp difference is 103793584 for nearest: 1736213617199858628 - target: 1736213617096065044 at idx 245 +For topic gripper_width: timestamp difference is 105430588 for nearest: 1736213618799918637 - target: 1736213618694488049 at idx 249 +[INFO] [1736213626.988121]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736213627.757915]: Stored episode 3. +[INFO] [1736213627.761473]: Saving video... +[INFO] [1736213627.951274]: Video saved! +Starting episode 3 +[INFO] [1736213627.983554]: resetting environment +[INFO] [1736213627.991586]: cleared current plan +[INFO] [1736213627.991778]: moving to home +[INFO] [1736213631.697886]: reached home +[INFO] [1736213631.705622]: reset action history +[INFO] [1736213634.638228]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213635.960531]: resetting environment +[INFO] [1736213635.978120]: cleared current plan +[INFO] [1736213635.982497]: moving to home +[INFO] [1736213636.987381]: reached home +[INFO] [1736213636.989301]: reset action history +[INFO] [1736213639.705919]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/episode_rosbags/episode_3_2025-01-06-20-33-59.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2110680.log +started roslaunch server http://158.130.50.37:46315/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2111528] +process[print_text-11]: started with pid [2111529] +process[rosbag_record-12]: started with pid [2111530] +started bagging! +For topic gripper_width: timestamp difference is -109946832 for nearest: 1736213640433237210 - target: 1736213640543184042 at idx 14 +For topic gripper_width: timestamp difference is -107963448 for nearest: 1736213643433238143 - target: 1736213643541201591 at idx 53 +For topic gripper_width: timestamp difference is -103501186 for nearest: 1736213649633233920 - target: 1736213649736735106 at idx 138 +[INFO] [1736213666.804250]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736213667.580628]: Stored episode 4. +[INFO] [1736213667.590251]: Saving video... +[INFO] [1736213667.801618]: Video saved! +Starting episode 4 +[INFO] [1736213667.845724]: resetting environment +[INFO] [1736213667.856955]: cleared current plan +[INFO] [1736213667.860009]: moving to home +[INFO] [1736213671.669863]: reached home +[INFO] [1736213671.671599]: reset action history +[INFO] [1736213674.449233]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736213676.204771]: resetting environment +[INFO] [1736213676.221083]: cleared current plan +[INFO] [1736213676.231396]: moving to home +[INFO] [1736213677.238317]: reached home +[INFO] [1736213677.240474]: reset action history +[INFO] [1736213680.022421]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/episode_rosbags/episode_4_2025-01-06-20-34-40.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2110680.log +started roslaunch server http://158.130.50.37:34705/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2111664] +process[print_text-14]: started with pid [2111665] +process[rosbag_record-15]: started with pid [2111666] +started bagging! +For topic gripper_width: timestamp difference is -178651276 for nearest: 1736213690933234033 - target: 1736213691111885309 at idx 148 +[INFO] [1736213707.166234]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736213707.966533]: Stored episode 5. +[INFO] [1736213707.969893]: Saving video... +[INFO] [1736213708.168067]: Video saved! +[INFO] [1736213708.220284]: resetting environment +[INFO] [1736213708.239447]: cleared current plan +[INFO] [1736213708.239879]: moving to home +[INFO] [1736213712.050314]: reached home +[INFO] [1736213712.055732]: reset action history +[INFO] [1736213714.845293]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/requirements.txt b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-metadata.json b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b530e8cb2e12b0c5c524bc5b918e0948a4d4b42b --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T01:31:32.116005", + "startedAt": "2025-01-07T01:31:31.760526", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4499.796625000001, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.014, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.004, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.022, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 498.98254013061523 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..402c7b305e0f3daa310fcf56c0cfb1be560ad25d --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c19b9ddad84cf3e30386668b1ce4f00bca8b34974ab73f345354a77fd2484162", "size": 1143520, "path": "media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4"}, "global_step": 664, "_timestamp": 1736213721.3178725, "_runtime": 229.54798460006714, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "2e671e1fc9c3432634515aa8a65c79d3612fb9df1d6b9e5ee0a85c2d2eededc6", "size": 1149197, "path": "media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4"}, "eval/num_success": 5.0, "episode": 15.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "4da772e21c938b47e3f0aee289e1b2029e73f393a2ce93590b0859f47887b948", "size": 1150379, "path": "media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "d4c60367d0b4d6cdfda4ef568547dad9d0fe06f74a235ceea5cc5cb024bb956c", "size": 1149174, "path": "media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "24ed8d7ed14931d68e7d126977166aacc6caf913f3829e6c967dd00e1822f1d8", "size": 1149294, "path": "media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://z1xjsysa4vgz8f3s9or4wczjoh0y0l332hduc3rx3z89bbogqr9chjxk5hxsax0edrx0eyh0105gy4ccds4u2ngsj2o508swo0u2xmorx66zc9k0cr5p8zwgnz1jzo41/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://tskme68azz33htmzia3srykr1kgcjvld4oapta9a5h5bi9r9tz9vds0odn0d0nke21kkvf29n6gi140qly6kangtmjtzv9s4efadhx8cr9z6f0cp254137ts83amsb5p:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 5, "total_num_episodes": 15, "total_success_rate": 0.3333333333333333, "total_success_rate_lb_0.01": 0.10265967257817107, "total_success_rate_ub_0.01": 0.6413327455237964, "total_success_rate_lb_0.025": 0.14299744374739537, "total_success_rate_ub_0.025": 0.6138283610116277, "total_success_rate_lb_0.05": 0.1682697982259454, "total_success_rate_ub_0.05": 0.5397089421669697, "total_success_rate_lb_0.1": 0.17985042190994288, "total_success_rate_ub_0.1": 0.5254023298570373, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "85e3ce6114f85ccb0921eb8f14c4ad7f5799f142a0d0fe166713ff8aac5cd801", "size": 522, "artifact_path": "wandb-client-artifact://6w780nkn8irinu3rrsh1sv38e5dl01wfwa83o7eoio9ny0hf0ncvaluod3nmewtivco4i2c65p6n6f7qptzxt91j2vsu9tiipcsrzbd6ty4bf0wjopnlbznqsja9ndab/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://i2tqyfj6pqyr28rmtqwqzkjw1tvf5uxu44x37h1rsa1h8mfpn4j9381pykbss4kojhj78nlneshrgbloqwdrwdbafvj8aay1u2l7gdxcfz48gr7hyczjiaygybqmcoa5:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 240}, "eval/total_success_rate": 0.3333333432674408, "eval/total_success_rate_lb_0.01": 0.102659672498703, "eval/total_success_rate_ub_0.01": 0.641332745552063, "eval/total_success_rate_lb_0.025": 0.14299744367599487, "eval/total_success_rate_ub_0.025": 0.6138283610343933, "eval/total_success_rate_lb_0.05": 0.1682697981595993, "eval/total_success_rate_ub_0.05": 0.5397089123725891, "eval/total_success_rate_lb_0.1": 0.17985042929649353, "eval/total_success_rate_ub_0.1": 0.525402307510376} \ No newline at end of file diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug-internal.log b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..255e791a29cf971a8573193d8ddd483ccbd831c6 --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug-internal.log @@ -0,0 +1,732 @@ +2025-01-06 20:31:31,769 INFO StreamThr :2110981 [internal.py:wandb_internal():85] W&B internal server running at pid: 2110981, started at: 2025-01-06 20:31:31.769206 +2025-01-06 20:31:31,770 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status +2025-01-06 20:31:31,782 INFO WriterThread:2110981 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/run-jwgj4azt.wandb +2025-01-06 20:31:31,782 DEBUG SenderThread:2110981 [sender.py:send():379] send: header +2025-01-06 20:31:31,792 DEBUG SenderThread:2110981 [sender.py:send():379] send: run +2025-01-06 20:31:32,024 INFO SenderThread:2110981 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files +2025-01-06 20:31:32,024 INFO SenderThread:2110981 [sender.py:_start_run_threads():1188] run started: jwgj4azt with start time 1736213491.769888 +2025-01-06 20:31:32,037 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 20:31:32,038 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: check_version +2025-01-06 20:31:32,096 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 20:31:32,101 DEBUG HandlerThread:2110981 [system_info.py:__init__():26] System info init +2025-01-06 20:31:32,101 DEBUG HandlerThread:2110981 [system_info.py:__init__():41] System info init done +2025-01-06 20:31:32,101 INFO HandlerThread:2110981 [system_monitor.py:start():194] Starting system monitor +2025-01-06 20:31:32,101 INFO SystemMonitor:2110981 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 20:31:32,101 INFO HandlerThread:2110981 [system_monitor.py:probe():214] Collecting system info +2025-01-06 20:31:32,102 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 20:31:32,102 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started disk monitoring +2025-01-06 20:31:32,103 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 20:31:32,103 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started memory monitoring +2025-01-06 20:31:32,103 INFO SystemMonitor:2110981 [interfaces.py:start():188] Started network monitoring +2025-01-06 20:31:32,115 DEBUG HandlerThread:2110981 [system_info.py:probe():152] Probing system +2025-01-06 20:31:32,117 DEBUG HandlerThread:2110981 [system_info.py:_probe_git():137] Probing git +2025-01-06 20:31:32,122 DEBUG HandlerThread:2110981 [system_info.py:_probe_git():145] Probing git done +2025-01-06 20:31:32,122 DEBUG HandlerThread:2110981 [system_info.py:probe():200] Probing system done +2025-01-06 20:31:32,122 DEBUG HandlerThread:2110981 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T01:31:32.116005', 'startedAt': '2025-01-07T01:31:31.760526', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4499.796625000001, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.014, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.004, 'min': 800.0, 'max': 4700.0}, {'current': 4500.022, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 498.98254013061523}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 20:31:32,122 INFO HandlerThread:2110981 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 20:31:32,122 INFO HandlerThread:2110981 [system_monitor.py:probe():227] Publishing system info +2025-01-06 20:31:32,122 DEBUG HandlerThread:2110981 [system_info.py:_save_code():44] Saving code +2025-01-06 20:31:32,124 DEBUG HandlerThread:2110981 [system_info.py:_save_code():67] Saving code done +2025-01-06 20:31:32,124 DEBUG HandlerThread:2110981 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 20:31:32,186 DEBUG HandlerThread:2110981 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 20:31:32,187 INFO HandlerThread:2110981 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 20:31:32,190 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:31:32,190 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 20:31:32,190 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 20:31:32,190 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 20:31:32,439 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/3d21j0cb-wandb-metadata.json +2025-01-06 20:31:32,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 20:31:32,444 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: python_packages +2025-01-06 20:31:32,445 DEBUG SenderThread:2110981 [sender.py:send():379] send: telemetry +2025-01-06 20:31:32,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:32,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:31:32,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:31:32,448 INFO HandlerThread:2110981 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/tb" + save: true +} +control { + relay_id: "7ce92a542acc48e38ea9e8d4834a33f7" +} +_info { + stream_id: "jwgj4azt" +} + +2025-01-06 20:31:32,449 DEBUG HandlerThread:2110981 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 20:31:32,475 INFO wandb-upload_1:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/5odjnous-code/FISH/eval_robot.py +2025-01-06 20:31:32,515 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/y6oe653p-diff.patch +2025-01-06 20:31:32,539 DEBUG SenderThread:2110981 [sender.py:send():379] send: tbrecord +2025-01-06 20:31:32,539 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:31:32,539 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 with policy live +2025-01-06 20:31:32,734 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/i5kuvii2-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:31:33,026 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-metadata.json +2025-01-06 20:31:33,026 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/diff.patch +2025-01-06 20:31:33,026 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code/FISH/eval_robot.py +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/requirements.txt +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code +2025-01-06 20:31:33,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code/FISH +2025-01-06 20:31:33,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:34,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:35,025 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:35,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:36,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:37,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:37,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:38,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:39,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:40,165 DEBUG SenderThread:2110981 [sender.py:send():379] send: config +2025-01-06 20:31:40,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:41,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:42,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:42,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:43,027 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:43,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:44,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:45,028 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:45,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:46,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:47,029 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:47,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:31:47,444 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:31:47,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:47,536 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:48,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:49,029 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:49,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:50,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:51,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:52,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:53,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:53,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:54,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:55,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:56,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:57,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:58,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:31:58,693 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:31:59,032 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:31:59,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:00,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:01,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:02,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:32:02,444 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:32:02,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:03,033 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:03,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:04,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:04,536 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:05,034 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml +2025-01-06 20:32:05,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:06,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:07,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:08,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:09,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:10,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:10,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:11,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:12,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:13,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:14,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:15,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:15,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:16,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:17,444 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:32:17,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:32:17,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:18,450 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:19,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:20,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:20,551 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:21,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:22,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:23,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:24,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:25,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:26,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:26,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:27,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:28,325 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:32:28,325 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 with policy now +2025-01-06 20:32:28,325 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:32:28,326 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:32:28,326 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:32:28,327 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:32:28,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:28,749 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/jfz6y511-media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 +2025-01-06 20:32:29,040 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media +2025-01-06 20:32:29,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos +2025-01-06 20:32:29,316 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/l0of5mt0-media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 +2025-01-06 20:32:29,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:30,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:31,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:31,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:32,033 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:32,103 DEBUG SystemMonitor:2110981 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 20:32:32,104 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:32:32,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:32:32,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:32:32,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:33,041 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:33,454 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:34,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:35,454 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:36,042 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:32:36,301 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/fn7ymk3h-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:32:36,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:37,043 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:37,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:37,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:38,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:39,044 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:39,453 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:40,454 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:41,044 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:41,459 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:42,458 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:43,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:43,458 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:44,458 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:45,459 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:46,459 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:47,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:32:47,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:32:47,489 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:48,459 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:48,532 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:49,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:50,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:51,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:52,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:53,048 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:53,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:53,765 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:54,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:55,049 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:32:55,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:56,464 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:57,463 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:58,463 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:32:59,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:32:59,464 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:00,464 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:01,464 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:02,105 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:33:02,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:33:02,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:33:02,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:03,465 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:04,465 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:04,547 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:05,472 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:06,471 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:07,493 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:07,697 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:33:07,698 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:33:07,698 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 with policy now +2025-01-06 20:33:07,698 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:33:07,698 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:33:07,699 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:33:07,969 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/nj9q2ipb-media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 +2025-01-06 20:33:08,053 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:33:08,053 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 +2025-01-06 20:33:08,053 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:33:08,312 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/5et240k5-media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 +2025-01-06 20:33:08,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:09,053 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:09,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:09,700 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:10,474 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:11,473 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:12,479 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:13,054 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:13,479 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:14,474 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:15,321 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:15,474 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:16,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:16,863 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:33:16,863 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:33:16,870 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:33:17,056 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:17,056 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:33:17,056 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:33:17,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:33:17,449 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:33:17,479 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:18,484 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:19,056 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:19,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:20,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:21,057 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:21,275 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:21,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:22,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:23,058 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:23,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:24,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:25,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:26,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:26,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:27,475 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:28,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:29,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:30,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:31,449 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:31,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:32,105 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:33:32,445 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:33:32,445 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:33:32,489 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:33,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:34,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:35,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:36,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:36,607 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:37,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:38,476 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:39,063 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:39,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:40,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:41,064 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:41,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:41,857 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:42,477 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:43,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:44,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:45,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:46,478 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:46,988 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:47,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:33:47,447 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:33:47,481 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:47,979 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:33:47,979 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 with policy now +2025-01-06 20:33:47,981 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:33:47,982 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:33:47,982 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:33:47,983 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:33:48,066 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:33:48,066 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 +2025-01-06 20:33:48,066 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:33:48,268 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/s5sq8j85-media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 +2025-01-06 20:33:48,481 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:48,528 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/50aovh5p-media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 +2025-01-06 20:33:49,066 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:49,484 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:50,483 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:51,483 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:52,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:52,485 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:53,068 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:53,483 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:54,616 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:55,613 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:56,069 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:33:56,313 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/3wmnsj9n-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:33:56,615 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:56,694 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:33:56,694 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:33:56,694 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:33:57,069 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:57,070 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:33:57,454 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:33:57,614 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:58,618 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:33:59,070 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:33:59,629 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:00,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:01,071 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:01,618 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:02,106 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:34:02,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:34:02,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:34:02,576 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:02,618 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:03,071 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:03,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:04,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:05,072 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:05,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:06,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:07,619 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:08,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:08,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:09,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:10,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:11,074 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:11,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:12,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:13,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:14,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:14,620 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:15,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:16,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:17,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:34:17,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:34:17,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:18,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:19,594 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:19,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:20,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:21,621 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:22,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:23,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:24,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:25,452 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:25,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:26,622 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:27,627 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:27,845 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:34:27,845 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:34:27,846 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 with policy now +2025-01-06 20:34:27,846 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:34:27,846 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:34:27,847 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:34:28,080 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:34:28,080 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 +2025-01-06 20:34:28,080 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:34:28,119 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/d98zxpk2-media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 +2025-01-06 20:34:28,413 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/le7fj6rq-media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 +2025-01-06 20:34:28,634 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:29,080 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:29,641 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:30,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:30,860 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:31,640 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:32,106 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:34:32,448 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:34:32,449 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:34:32,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:33,081 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:33,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:34,640 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:35,641 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:36,163 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:36,647 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:36,743 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:34:36,743 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:34:36,743 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:34:37,083 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:37,083 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:34:37,083 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:34:37,649 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:38,649 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:39,084 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:39,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:40,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:41,084 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:41,466 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:41,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:42,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:43,085 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:43,642 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:44,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:45,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:46,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:47,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:34:47,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:34:47,537 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:47,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:48,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:49,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:50,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:51,643 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:52,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:53,088 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:34:53,457 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:53,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:54,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:55,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:56,644 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:57,647 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:58,457 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:34:58,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:34:59,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:00,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:01,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:02,106 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:35:02,446 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:35:02,446 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:35:02,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:03,568 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:03,645 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:04,646 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:05,646 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:06,646 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:07,653 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:08,215 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:35:08,215 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 with policy now +2025-01-06 20:35:08,219 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:35:08,220 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:08,220 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:08,221 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:08,486 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/mhwvxn89-media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 +2025-01-06 20:35:08,650 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:09,094 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:09,094 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:09,094 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 +2025-01-06 20:35:09,095 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval +2025-01-06 20:35:09,241 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:09,397 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/f9ubcnjy-media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 +2025-01-06 20:35:09,649 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:10,655 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:11,656 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:12,667 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:13,095 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:13,659 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:14,470 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:14,664 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:15,655 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:16,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:17,097 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:17,451 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:35:17,451 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:35:17,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:18,665 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:19,597 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:19,667 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:20,665 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:20,796 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:20,796 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:20,797 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:20,851 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:35:20,851 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:35:21,024 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:35:21,098 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:21,098 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:21,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:35:21,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media +2025-01-06 20:35:21,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval +2025-01-06 20:35:21,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table +2025-01-06 20:35:21,291 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/kuj5jjbj-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:35:21,347 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/ksk4cpuz-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:21,362 INFO wandb-upload_1:2110981 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0q7m3kfb +2025-01-06 20:35:21,420 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 20:35:21,429 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 20:35:21,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:22,030 INFO SenderThread:2110981 [sender.py:send_request_log_artifact():1525] logged artifact run-jwgj4azt-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI5MDE0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYzMjAyMQ==', 'latestArtifact': None}} +2025-01-06 20:35:22,030 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:35:22,030 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-06 20:35:22,031 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:22,031 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,031 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,031 DEBUG SenderThread:2110981 [sender.py:send():379] send: telemetry +2025-01-06 20:35:22,031 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,035 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,035 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,036 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,036 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,036 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,036 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,037 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,037 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,037 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,037 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,038 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,038 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,038 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,038 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,038 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,039 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,039 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,039 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,039 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,040 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,040 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:22,040 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 20:35:22,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:22,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:22,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json +2025-01-06 20:35:22,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval +2025-01-06 20:35:22,289 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/d881s9ye-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:35:22,293 INFO wandb-upload_1:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/ey9moiko-media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json +2025-01-06 20:35:22,435 INFO wandb-upload_2:2110981 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp4ks2i1aa +2025-01-06 20:35:22,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:22,898 INFO SenderThread:2110981 [sender.py:send_request_log_artifact():1525] logged artifact run-jwgj4azt-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjI5MDIzOQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDYzMjEwMQ==', 'latestArtifact': None}} +2025-01-06 20:35:22,898 DEBUG SenderThread:2110981 [sender.py:send():379] send: files +2025-01-06 20:35:22,898 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json with policy now +2025-01-06 20:35:22,898 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:22,898 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:22,899 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:23,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:23,099 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:23,131 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/xozmhro4-media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json +2025-01-06 20:35:23,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:24,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:24,899 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:25,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:26,662 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:27,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:28,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:29,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:30,460 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:30,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:31,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:32,107 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:35:32,447 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 20:35:32,447 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: stop_status +2025-01-06 20:35:32,663 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:32,925 DEBUG SenderThread:2110981 [sender.py:send():379] send: exit +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:send_exit():588] handling runtime: 240 +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:send_exit():594] send defer +2025-01-06 20:35:32,926 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,926 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 20:35:32,926 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 20:35:32,926 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 1 +2025-01-06 20:35:32,927 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,927 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 20:35:32,927 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:32,927 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 20:35:32,927 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 2 +2025-01-06 20:35:32,927 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,927 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 20:35:32,927 INFO HandlerThread:2110981 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 20:35:32,927 DEBUG SystemMonitor:2110981 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 20:35:32,927 DEBUG SystemMonitor:2110981 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 20:35:32,928 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 20:35:32,928 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 20:35:32,979 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 20:35:32,979 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 20:35:32,979 INFO HandlerThread:2110981 [interfaces.py:finish():200] Joined network monitor +2025-01-06 20:35:32,980 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:32,980 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 20:35:32,980 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 3 +2025-01-06 20:35:32,980 DEBUG SenderThread:2110981 [sender.py:send():379] send: stats +2025-01-06 20:35:32,980 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,980 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 20:35:32,980 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:32,980 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 20:35:32,980 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 4 +2025-01-06 20:35:32,980 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:32,980 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 20:35:33,103 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:35,104 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:39,802 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:39,802 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:39,802 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 20:35:39,803 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:39,803 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 20:35:39,803 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 5 +2025-01-06 20:35:39,803 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:39,804 DEBUG SenderThread:2110981 [sender.py:send():379] send: history +2025-01-06 20:35:39,804 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: summary_record +2025-01-06 20:35:39,804 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:39,804 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 20:35:39,804 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:39,805 DEBUG SenderThread:2110981 [sender.py:send():379] send: summary +2025-01-06 20:35:39,805 INFO SenderThread:2110981 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 20:35:39,805 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:39,805 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 20:35:39,805 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 6 +2025-01-06 20:35:39,805 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:39,806 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 20:35:39,806 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:39,806 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 20:35:39,809 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 20:35:39,926 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 7 +2025-01-06 20:35:39,927 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:39,927 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 20:35:39,927 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:39,927 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 20:35:39,927 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:40,106 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:40,106 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml +2025-01-06 20:35:40,789 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 8 +2025-01-06 20:35:40,789 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:40,789 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:40,789 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 20:35:40,790 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:40,790 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 20:35:40,790 INFO SenderThread:2110981 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 20:35:40,792 INFO SenderThread:2110981 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 20:35:40,794 INFO SenderThread:2110981 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 20:35:40,811 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 9 +2025-01-06 20:35:40,811 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:40,811 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 20:35:40,811 DEBUG SenderThread:2110981 [sender.py:send():379] send: artifact +2025-01-06 20:35:40,928 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:41,013 INFO SenderThread:2110981 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 20:35:41,013 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,014 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 20:35:41,014 INFO SenderThread:2110981 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 20:35:41,106 INFO Thread-12 :2110981 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml config.yaml +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json wandb-summary.json +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:41,107 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/requirements.txt requirements.txt +2025-01-06 20:35:41,108 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-metadata.json wandb-metadata.json +2025-01-06 20:35:41,108 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log output.log +2025-01-06 20:35:41,108 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/diff.patch diff.patch +2025-01-06 20:35:41,110 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 20:35:41,112 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 media/videos/eval/4_eval_4_24ed8d7ed14931d68e7d.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 media/videos/eval/3_eval_3_d4c60367d0b4d6cdfda4.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 media/videos/eval/2_eval_2_4da772e21c938b47e3f0.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 media/videos/eval/0_eval_0_c19b9ddad84cf3e30386.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 media/videos/eval/1_eval_1_2e671e1fc9c343263451.mp4 +2025-01-06 20:35:41,116 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json media/table/eval/total_success_rate_ci_6_85e3ce6114f85ccb0921.table.json +2025-01-06 20:35:41,117 INFO SenderThread:2110981 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-06 20:35:41,117 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 10 +2025-01-06 20:35:41,117 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:41,117 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,117 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 20:35:41,117 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,117 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 20:35:41,117 INFO SenderThread:2110981 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:35:41,288 INFO wandb-upload_2:2110981 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/config.yaml +2025-01-06 20:35:41,298 INFO wandb-upload_1:2110981 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/wandb-summary.json +2025-01-06 20:35:41,362 INFO wandb-upload_4:2110981 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/requirements.txt +2025-01-06 20:35:41,376 INFO wandb-upload_3:2110981 [upload_job.py:push():130] Uploaded file /tmp/tmpmvrisps2wandb/7gebbo7f-events.out.tfevents.1736213492.leonmkim-ROG-Strix-G15CS-G15CS.2110680.0 +2025-01-06 20:35:41,379 INFO wandb-upload_0:2110981 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/files/output.log +2025-01-06 20:35:41,580 INFO Thread-11 (_thread_body):2110981 [sender.py:transition_state():614] send defer: 11 +2025-01-06 20:35:41,580 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,580 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 20:35:41,580 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,580 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 20:35:41,580 INFO SenderThread:2110981 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 20:35:41,581 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 12 +2025-01-06 20:35:41,582 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,582 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 20:35:41,582 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,582 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 20:35:41,582 INFO SenderThread:2110981 [file_stream.py:finish():601] file stream finish called +2025-01-06 20:35:41,805 INFO SenderThread:2110981 [file_stream.py:finish():605] file stream finish is done +2025-01-06 20:35:41,805 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 13 +2025-01-06 20:35:41,805 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,805 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 20:35:41,806 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,806 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 20:35:41,806 INFO SenderThread:2110981 [sender.py:transition_state():614] send defer: 14 +2025-01-06 20:35:41,806 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: defer +2025-01-06 20:35:41,806 INFO HandlerThread:2110981 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 20:35:41,806 DEBUG SenderThread:2110981 [sender.py:send():379] send: final +2025-01-06 20:35:41,806 DEBUG SenderThread:2110981 [sender.py:send():379] send: footer +2025-01-06 20:35:41,806 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: defer +2025-01-06 20:35:41,806 INFO SenderThread:2110981 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 20:35:41,807 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:41,807 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:41,807 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 20:35:41,807 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 20:35:41,808 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 20:35:41,808 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 20:35:41,808 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: server_info +2025-01-06 20:35:41,810 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 20:35:41,860 DEBUG SenderThread:2110981 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 20:35:41,860 INFO MainThread:2110981 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 20:35:41,860 INFO MainThread:2110981 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 20:35:41,860 INFO MainThread:2110981 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 20:35:41,861 DEBUG HandlerThread:2110981 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 20:35:41,861 INFO HandlerThread:2110981 [handler.py:finish():882] shutting down handler +2025-01-06 20:35:42,808 INFO WriterThread:2110981 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/run-jwgj4azt.wandb +2025-01-06 20:35:42,860 INFO SenderThread:2110981 [sender.py:finish():1615] shutting down sender +2025-01-06 20:35:42,860 INFO SenderThread:2110981 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 20:35:42,860 INFO SenderThread:2110981 [file_pusher.py:join():175] waiting for file pusher diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug.log b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d3ff7a1424750ac620034ef2043a8372bd23031b --- /dev/null +++ b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Configure stats pid to 2110680 +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/settings +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug.log +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/logs/debug-internal.log +2025-01-06 20:31:31,762 INFO MainThread:2110680 [wandb_init.py:init():569] calling init triggers +2025-01-06 20:31:31,763 INFO MainThread:2110680 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt'} +2025-01-06 20:31:31,763 INFO MainThread:2110680 [wandb_init.py:init():619] starting backend +2025-01-06 20:31:31,763 INFO MainThread:2110680 [wandb_init.py:init():623] setting up manager +2025-01-06 20:31:31,768 INFO MainThread:2110680 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 20:31:31,769 INFO MainThread:2110680 [wandb_init.py:init():631] backend started and connected +2025-01-06 20:31:31,781 INFO MainThread:2110680 [wandb_init.py:init():720] updated telemetry +2025-01-06 20:31:31,790 INFO MainThread:2110680 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 20:31:32,037 INFO MainThread:2110680 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 20:31:32,091 INFO MainThread:2110680 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 20:31:32,091 INFO MainThread:2110680 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 20:31:32,444 INFO MainThread:2110680 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 20:31:32,444 INFO MainThread:2110680 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 20:31:32,444 INFO MainThread:2110680 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 20:31:32,444 INFO MainThread:2110680 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 20:31:32,446 INFO MainThread:2110680 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 20:31:32,446 INFO MainThread:2110680 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/jwgj4azt/tb, True +2025-01-06 20:31:40,164 INFO MainThread:2110680 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'fowlers', 'left_book_slot': 'twodim'} +2025-01-06 20:35:42,861 WARNING MsgRouterThr:2110680 [router.py:message_loop():77] message_loop has been closed diff --git a/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/run-jwgj4azt.wandb b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/run-jwgj4azt.wandb new file mode 100644 index 0000000000000000000000000000000000000000..80567381388aa1532afac9dc268f0afb6758df4d Binary files /dev/null and b/jwgj4azt/wandb/run-20250106_203131-jwgj4azt/run-jwgj4azt.wandb differ diff --git a/kq00a7xd/episode_rosbags/episode_2_2025-01-07-18-03-06.bag b/kq00a7xd/episode_rosbags/episode_2_2025-01-07-18-03-06.bag new file mode 100644 index 0000000000000000000000000000000000000000..64b31dac29707e80ea3c2655283b0032b979b071 --- /dev/null +++ b/kq00a7xd/episode_rosbags/episode_2_2025-01-07-18-03-06.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6637bdade4ced9752423e997f0c43bac65c5e42e372d2b5a677a208002e1649c +size 1412388442 diff --git a/kq00a7xd/episode_rosbags/episode_3_2025-01-07-18-03-47.bag b/kq00a7xd/episode_rosbags/episode_3_2025-01-07-18-03-47.bag new file mode 100644 index 0000000000000000000000000000000000000000..7f1e12f792c90021337fdad900822bd472f35210 --- /dev/null +++ b/kq00a7xd/episode_rosbags/episode_3_2025-01-07-18-03-47.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:473008f4b5781ab65f12998e797dc8834e46b9bf16bb4bd31adb226bc2fbb7aa +size 1414054098 diff --git a/snapshot_11342.pt b/snapshot_11342.pt new file mode 100644 index 0000000000000000000000000000000000000000..5df1d9c5601871c379bf4d13177eb7e6863464ad --- /dev/null +++ b/snapshot_11342.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd93ad7a3fd97d0bad52d4b3d4763de9d2a11eb9db02554c161bf84858df3ab8 +size 910102870 diff --git a/z78uirsa/.hydra/config.yaml b/z78uirsa/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/z78uirsa/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/z78uirsa/.hydra/hydra.yaml b/z78uirsa/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f128615cc93cd53890e2230c7756fcd3b04fd7e --- /dev/null +++ b/z78uirsa/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/z78uirsa + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/z78uirsa/.hydra/overrides.yaml b/z78uirsa/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/z78uirsa/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/z78uirsa/eval_policy.log b/z78uirsa/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..eeab643750081bed350dfcf13f2189b0ac02b045 --- /dev/null +++ b/z78uirsa/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 20:34:15,143][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:34:15,146][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 20:34:18,565][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) +