diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..6927441372355c96922e657d64d6fa29431c789f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,37 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +aq8qih16/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/episode_rosbags/episode_2_2025-01-21-21-32-52.bag filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/episode_rosbags/episode_2_2025-01-21-21-52-18.bag filter=lfs diff=lfs merge=lfs -text +q5zzaq6a/episode_rosbags/episode_1_2025-01-21-21-51-46.bag filter=lfs diff=lfs merge=lfs -text +pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +pkbouru1/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +1rhki7g4/episode_rosbags/episode_4_2025-01-21-22-25-50.bag filter=lfs diff=lfs merge=lfs -text diff --git a/1rhki7g4/episode_rosbags/aligned_depth_to_color_K.npy b/1rhki7g4/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/1rhki7g4/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/1rhki7g4/episode_rosbags/cam_tf_world.npy b/1rhki7g4/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/1rhki7g4/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/1rhki7g4/episode_rosbags/color_K.npy b/1rhki7g4/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/1rhki7g4/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/1rhki7g4/episode_rosbags/depth_K.npy b/1rhki7g4/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/1rhki7g4/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/1rhki7g4/episode_rosbags/episode_4_2025-01-21-22-25-50.bag b/1rhki7g4/episode_rosbags/episode_4_2025-01-21-22-25-50.bag new file mode 100644 index 0000000000000000000000000000000000000000..2ac69d6dff75573bb933b67b530379f16da3fc67 --- /dev/null +++ b/1rhki7g4/episode_rosbags/episode_4_2025-01-21-22-25-50.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce8034a4d23ddf4e0b03945af303682cf2cbaa8efd6a0bd869c1011becc2d97c +size 1559835220 diff --git a/1rhki7g4/eval_robot.log b/1rhki7g4/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..7bcb2092acfe2363da91fd90320a16ff2646ba0e --- /dev/null +++ b/1rhki7g4/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 22:22:13,130][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 22:22:13,133][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/1rhki7g4/wandb/debug-internal.log b/1rhki7g4/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c5ab246e3d11ee60bdae8fc5639a3059b9b4fde7 --- /dev/null +++ b/1rhki7g4/wandb/debug-internal.log @@ -0,0 +1,773 @@ +2025-01-21 22:22:18,909 INFO StreamThr :2665561 [internal.py:wandb_internal():85] W&B internal server running at pid: 2665561, started at: 2025-01-21 22:22:18.909069 +2025-01-21 22:22:18,910 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status +2025-01-21 22:22:18,924 INFO WriterThread:2665561 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/run-1rhki7g4.wandb +2025-01-21 22:22:18,924 DEBUG SenderThread:2665561 [sender.py:send():379] send: header +2025-01-21 22:22:18,932 DEBUG SenderThread:2665561 [sender.py:send():379] send: run +2025-01-21 22:22:19,181 INFO SenderThread:2665561 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files +2025-01-21 22:22:19,181 INFO SenderThread:2665561 [sender.py:_start_run_threads():1188] run started: 1rhki7g4 with start time 1737516138.909506 +2025-01-21 22:22:19,196 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 22:22:19,196 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: check_version +2025-01-21 22:22:19,267 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 22:22:19,272 DEBUG HandlerThread:2665561 [system_info.py:__init__():26] System info init +2025-01-21 22:22:19,272 DEBUG HandlerThread:2665561 [system_info.py:__init__():41] System info init done +2025-01-21 22:22:19,272 INFO HandlerThread:2665561 [system_monitor.py:start():194] Starting system monitor +2025-01-21 22:22:19,272 INFO SystemMonitor:2665561 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 22:22:19,272 INFO HandlerThread:2665561 [system_monitor.py:probe():214] Collecting system info +2025-01-21 22:22:19,273 INFO SystemMonitor:2665561 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 22:22:19,274 INFO SystemMonitor:2665561 [interfaces.py:start():188] Started disk monitoring +2025-01-21 22:22:19,274 INFO SystemMonitor:2665561 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 22:22:19,275 INFO SystemMonitor:2665561 [interfaces.py:start():188] Started memory monitoring +2025-01-21 22:22:19,275 INFO SystemMonitor:2665561 [interfaces.py:start():188] Started network monitoring +2025-01-21 22:22:19,286 DEBUG HandlerThread:2665561 [system_info.py:probe():152] Probing system +2025-01-21 22:22:19,288 DEBUG HandlerThread:2665561 [system_info.py:_probe_git():137] Probing git +2025-01-21 22:22:19,292 DEBUG HandlerThread:2665561 [system_info.py:_probe_git():145] Probing git done +2025-01-21 22:22:19,292 DEBUG HandlerThread:2665561 [system_info.py:probe():200] Probing system done +2025-01-21 22:22:19,292 DEBUG HandlerThread:2665561 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T03:22:19.286582', 'startedAt': '2025-01-22T03:22:18.897530', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.5865, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.929, 'min': 800.0, 'max': 4700.0}, {'current': 4492.694, 'min': 800.0, 'max': 4700.0}, {'current': 4499.999, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 477.7547721862793}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 22:22:19,292 INFO HandlerThread:2665561 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 22:22:19,292 INFO HandlerThread:2665561 [system_monitor.py:probe():227] Publishing system info +2025-01-21 22:22:19,292 DEBUG HandlerThread:2665561 [system_info.py:_save_code():44] Saving code +2025-01-21 22:22:19,294 DEBUG HandlerThread:2665561 [system_info.py:_save_code():67] Saving code done +2025-01-21 22:22:19,294 DEBUG HandlerThread:2665561 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 22:22:19,356 DEBUG HandlerThread:2665561 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 22:22:19,356 INFO HandlerThread:2665561 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 22:22:19,359 DEBUG SenderThread:2665561 [sender.py:send():379] send: files +2025-01-21 22:22:19,359 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 22:22:19,360 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 22:22:19,360 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 22:22:19,612 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 22:22:19,612 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: python_packages +2025-01-21 22:22:19,613 DEBUG SenderThread:2665561 [sender.py:send():379] send: telemetry +2025-01-21 22:22:19,614 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/rdvjh56r-wandb-metadata.json +2025-01-21 22:22:19,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:19,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:22:19,615 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:22:19,617 INFO wandb-upload_2:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/wbb4qncs-diff.patch +2025-01-21 22:22:19,657 INFO HandlerThread:2665561 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/tb" + save: true +} +control { + relay_id: "0c2631e3c9a44ac99e4c05690f671b85" +} +_info { + stream_id: "1rhki7g4" +} + +2025-01-21 22:22:19,657 DEBUG HandlerThread:2665561 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 22:22:19,659 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/3zt54qfp-code/FISH/eval_robot.py +2025-01-21 22:22:19,707 DEBUG SenderThread:2665561 [sender.py:send():379] send: tbrecord +2025-01-21 22:22:19,710 DEBUG SenderThread:2665561 [sender.py:send():379] send: files +2025-01-21 22:22:19,710 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 with policy live +2025-01-21 22:22:19,900 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/ve3hyi2l-events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:22:20,182 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:22:20,182 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/requirements.txt +2025-01-21 22:22:20,183 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/diff.patch +2025-01-21 22:22:20,183 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:20,183 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-metadata.json +2025-01-21 22:22:20,183 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/code/FISH/eval_robot.py +2025-01-21 22:22:20,183 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/code/FISH +2025-01-21 22:22:20,183 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/code +2025-01-21 22:22:20,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:21,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:22,182 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:22,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:23,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:24,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:22:24,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:25,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:26,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:27,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:28,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:29,135 DEBUG SenderThread:2665561 [sender.py:send():379] send: config +2025-01-21 22:22:29,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:22:29,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:30,184 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:30,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:31,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:32,184 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:32,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:33,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:34,185 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:34,612 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:22:34,613 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:22:34,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:34,722 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:22:35,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:36,186 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:36,625 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:37,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:38,186 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:38,616 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:39,616 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:40,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:22:40,616 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:41,617 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:42,617 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:43,617 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:44,188 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:44,617 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:45,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:22:45,617 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:46,188 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:46,617 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:47,625 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:48,189 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:48,625 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:49,612 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:22:49,613 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:22:49,657 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:50,625 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:50,702 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:22:51,190 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/config.yaml +2025-01-21 22:22:51,626 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:52,190 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:22:52,626 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:53,626 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:54,626 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:55,626 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:56,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:22:56,626 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:57,627 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:58,627 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:22:59,627 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:00,627 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:01,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:01,627 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:02,627 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:03,628 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:04,612 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:23:04,613 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:23:04,653 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:05,628 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:06,628 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:06,697 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:07,628 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:08,628 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:09,629 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:10,629 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:11,631 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:12,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:12,629 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:13,629 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:14,630 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:15,630 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:16,632 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:17,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:17,631 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:18,631 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:19,276 DEBUG SystemMonitor:2665561 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 22:23:19,276 DEBUG SenderThread:2665561 [sender.py:send():379] send: stats +2025-01-21 22:23:19,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:23:19,613 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:23:19,653 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:20,631 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:21,631 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:22,632 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:22,731 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:22,968 DEBUG SenderThread:2665561 [sender.py:send():379] send: files +2025-01-21 22:23:22,968 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_6d4609e89d3ba1e1c763.mp4 with policy now +2025-01-21 22:23:22,969 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:23:22,970 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:23:22,970 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:23:22,970 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:23:23,200 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/0_eval_0_6d4609e89d3ba1e1c763.mp4 +2025-01-21 22:23:23,200 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:23:23,200 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos +2025-01-21 22:23:23,200 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval +2025-01-21 22:23:23,201 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media +2025-01-21 22:23:23,230 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/fyt8ej8t-media/videos/eval/0_eval_0_6d4609e89d3ba1e1c763.mp4 +2025-01-21 22:23:23,514 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/pnu50j3f-media/videos/eval/0_eval_0_6d4609e89d3ba1e1c763.mp4 +2025-01-21 22:23:23,632 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:24,200 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:23:24,632 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:25,633 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:26,633 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:27,632 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:28,202 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:23:28,629 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:28,634 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:29,633 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:30,633 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:31,203 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:23:31,430 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/hs5nz8ux-events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:23:31,633 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:32,203 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:23:32,635 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:33,635 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:33,821 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:34,204 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:23:34,613 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:23:34,614 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:23:34,636 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:35,635 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:36,205 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:23:36,636 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:37,635 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:38,205 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:23:38,635 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:39,616 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:39,636 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:40,636 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:41,636 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:42,637 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:43,640 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:44,616 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:44,640 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:45,638 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:46,638 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:47,639 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:48,641 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:49,277 DEBUG SenderThread:2665561 [sender.py:send():379] send: stats +2025-01-21 22:23:49,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:23:49,615 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:23:49,657 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:49,717 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:50,642 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:51,642 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:52,642 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:53,642 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:54,211 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:23:54,642 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:55,617 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:23:55,650 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:56,645 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:57,646 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:58,646 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:23:59,646 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:00,646 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:01,617 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:01,646 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:02,646 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:03,646 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:04,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:24:04,614 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:24:04,657 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:05,647 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:06,647 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:06,733 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:07,649 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:08,215 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:24:08,649 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:09,648 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:10,649 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:11,000 DEBUG SenderThread:2665561 [sender.py:send():379] send: files +2025-01-21 22:24:11,000 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_e6e0be02eedcefa21d59.mp4 with policy now +2025-01-21 22:24:11,001 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:24:11,002 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:24:11,002 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:24:11,077 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:24:11,216 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:24:11,216 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/1_eval_1_e6e0be02eedcefa21d59.mp4 +2025-01-21 22:24:11,216 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval +2025-01-21 22:24:11,267 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/e9wgkhcb-media/videos/eval/1_eval_1_e6e0be02eedcefa21d59.mp4 +2025-01-21 22:24:11,562 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/xb36f6kr-media/videos/eval/1_eval_1_e6e0be02eedcefa21d59.mp4 +2025-01-21 22:24:11,652 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:12,077 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:12,216 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:24:12,655 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:13,658 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:14,655 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:15,653 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:16,218 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:24:16,655 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:17,411 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:17,652 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:18,657 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:19,277 DEBUG SenderThread:2665561 [sender.py:send():379] send: stats +2025-01-21 22:24:19,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:24:19,614 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:24:19,664 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:20,219 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:24:20,663 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:21,655 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:22,661 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:22,694 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:23,660 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:24,660 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:25,658 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:26,659 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:27,221 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:24:27,661 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:28,085 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:24:28,085 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:24:28,085 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:28,086 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:24:28,222 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:24:28,222 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:24:28,662 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:29,672 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:30,222 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:24:30,677 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:31,670 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:32,223 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:24:32,664 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:33,664 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:33,749 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:34,224 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:24:34,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:24:34,614 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:24:34,664 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:35,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:36,665 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:37,665 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:38,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:39,623 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:39,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:40,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:41,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:42,226 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:24:42,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:43,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:44,623 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:44,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:45,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:46,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:47,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:48,666 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:49,278 DEBUG SenderThread:2665561 [sender.py:send():379] send: stats +2025-01-21 22:24:49,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:24:49,614 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:24:49,667 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:49,685 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:50,667 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:51,667 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:52,667 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:53,667 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:54,667 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:55,623 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:24:55,668 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:56,668 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:57,668 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:58,668 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:24:59,668 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:00,624 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:00,668 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:01,668 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:02,668 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:03,669 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:04,234 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:04,614 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:25:04,615 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:25:04,669 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:05,669 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:05,709 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:06,674 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:07,203 DEBUG SenderThread:2665561 [sender.py:send():379] send: files +2025-01-21 22:25:07,203 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_941ccc9a5f52aef980c0.mp4 with policy now +2025-01-21 22:25:07,204 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:25:07,204 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:25:07,204 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:25:07,211 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:25:07,235 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:25:07,235 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/2_eval_2_941ccc9a5f52aef980c0.mp4 +2025-01-21 22:25:07,235 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval +2025-01-21 22:25:07,443 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/ua2pquml-media/videos/eval/2_eval_2_941ccc9a5f52aef980c0.mp4 +2025-01-21 22:25:07,670 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:07,718 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/1iqtgbmo-media/videos/eval/2_eval_2_941ccc9a5f52aef980c0.mp4 +2025-01-21 22:25:08,235 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:08,679 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:09,680 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:10,675 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:10,841 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:11,682 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:12,236 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:12,677 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:13,673 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:14,681 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:15,677 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:16,238 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:16,632 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:16,677 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:17,238 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:25:17,485 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/u7mlgb47-events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:25:17,677 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:17,921 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:25:17,921 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:25:17,922 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:25:18,239 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:25:18,239 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:18,682 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:19,278 DEBUG SenderThread:2665561 [sender.py:send():379] send: stats +2025-01-21 22:25:19,616 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:25:19,617 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:25:19,688 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:20,239 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:20,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:21,685 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:21,945 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:22,240 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:22,682 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:23,685 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:24,241 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:24,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:25,685 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:26,242 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:26,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:27,220 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:27,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:28,242 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:28,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:29,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:30,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:31,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:32,628 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:32,687 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:33,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:34,615 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:25:34,615 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:25:34,686 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:35,687 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:36,687 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:37,687 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:37,703 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:38,691 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:39,615 DEBUG SenderThread:2665561 [sender.py:send():379] send: files +2025-01-21 22:25:39,615 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_56c00d964e4ff84c9625.mp4 with policy now +2025-01-21 22:25:39,620 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:25:39,620 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:25:39,620 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:25:39,620 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:25:39,697 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:39,832 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/hk5p2zus-media/videos/eval/3_eval_3_56c00d964e4ff84c9625.mp4 +2025-01-21 22:25:40,246 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:25:40,247 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:40,247 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/3_eval_3_56c00d964e4ff84c9625.mp4 +2025-01-21 22:25:40,247 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval +2025-01-21 22:25:40,507 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/dvsosw8u-media/videos/eval/3_eval_3_56c00d964e4ff84c9625.mp4 +2025-01-21 22:25:40,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:41,691 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:42,247 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:42,698 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:43,269 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:43,707 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:44,248 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:44,711 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:45,698 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:46,703 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:47,249 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:25:47,701 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:47,959 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:25:47,959 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:25:47,959 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:25:48,249 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:25:48,249 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:48,704 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:49,089 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:49,279 DEBUG SenderThread:2665561 [sender.py:send():379] send: stats +2025-01-21 22:25:49,621 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:25:49,621 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:25:49,707 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:50,250 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:50,715 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:51,702 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:52,251 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:52,702 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:53,703 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:54,251 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:25:54,342 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:54,703 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:55,704 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:56,703 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:57,703 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:58,703 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:25:59,346 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:25:59,703 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:00,254 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:00,704 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:01,704 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:02,704 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:03,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:04,616 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:26:04,616 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:26:04,685 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:04,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:05,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:06,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:07,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:08,257 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:08,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:09,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:10,638 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:10,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:11,705 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:12,706 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:13,712 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:14,224 DEBUG SenderThread:2665561 [sender.py:send():379] send: files +2025-01-21 22:26:14,224 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_92faad0d5acbe7e41d66.mp4 with policy now +2025-01-21 22:26:14,229 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:26:14,230 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:26:14,230 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:14,230 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:14,259 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:26:14,259 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:14,259 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/4_eval_4_92faad0d5acbe7e41d66.mp4 +2025-01-21 22:26:14,259 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval +2025-01-21 22:26:14,451 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/lhtak89v-media/videos/eval/4_eval_4_92faad0d5acbe7e41d66.mp4 +2025-01-21 22:26:14,707 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:14,735 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/68f1wpqi-media/videos/eval/4_eval_4_92faad0d5acbe7e41d66.mp4 +2025-01-21 22:26:15,639 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:15,710 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:16,259 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:16,722 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:17,724 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:18,724 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:19,279 DEBUG SenderThread:2665561 [sender.py:send():379] send: stats +2025-01-21 22:26:19,619 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:26:19,619 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:26:19,717 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:20,261 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:20,665 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:20,721 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:21,261 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:26:21,519 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/whlanlzj-events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:26:21,718 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:22,002 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:26:22,002 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:22,002 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:22,262 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:26:22,262 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:26:22,262 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:22,283 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:26:22,283 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:26:22,527 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:26:22,701 INFO wandb-upload_0:2665561 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp2sik6cbi +2025-01-21 22:26:22,717 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:22,936 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:26:22,957 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:26:23,096 INFO SenderThread:2665561 [sender.py:send_request_log_artifact():1525] logged artifact run-1rhki7g4-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzczODY0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwODY2OA==', 'latestArtifact': None}} +2025-01-21 22:26:23,096 DEBUG SenderThread:2665561 [sender.py:send():379] send: files +2025-01-21 22:26:23,096 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e4f3d858401fb0f04765.table.json with policy now +2025-01-21 22:26:23,097 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:26:23,097 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,097 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,097 DEBUG SenderThread:2665561 [sender.py:send():379] send: telemetry +2025-01-21 22:26:23,097 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,098 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,098 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,100 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,100 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,100 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,100 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,101 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,101 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,101 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,101 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,102 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,102 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,102 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,102 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,103 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,103 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,103 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,103 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,104 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,104 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,104 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:23,104 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:26:23,262 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:26:23,262 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:26:23,262 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/table/eval/success_rate_ci_5_e4f3d858401fb0f04765.table.json +2025-01-21 22:26:23,263 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/table/eval/total_success_rate_ci_6_06392b86e92c89b6b251.table.json +2025-01-21 22:26:23,263 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media +2025-01-21 22:26:23,263 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/table/eval +2025-01-21 22:26:23,263 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/table +2025-01-21 22:26:23,280 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/v3h2jbo1-media/table/eval/success_rate_ci_5_e4f3d858401fb0f04765.table.json +2025-01-21 22:26:23,441 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/0mniuqw3-media/table/eval/total_success_rate_ci_6_06392b86e92c89b6b251.table.json +2025-01-21 22:26:23,506 INFO wandb-upload_2:2665561 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0mjtb29r +2025-01-21 22:26:23,550 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/a98ne09l-media/table/eval/success_rate_ci_5_e4f3d858401fb0f04765.table.json +2025-01-21 22:26:23,717 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:23,878 INFO SenderThread:2665561 [sender.py:send_request_log_artifact():1525] logged artifact run-1rhki7g4-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzczODY5MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwODY5OA==', 'latestArtifact': None}} +2025-01-21 22:26:23,878 DEBUG SenderThread:2665561 [sender.py:send():379] send: files +2025-01-21 22:26:23,878 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_06392b86e92c89b6b251.table.json with policy now +2025-01-21 22:26:23,879 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:26:23,879 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:23,879 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:24,163 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/0x85sqhg-media/table/eval/total_success_rate_ci_6_06392b86e92c89b6b251.table.json +2025-01-21 22:26:24,263 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:26:24,263 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:24,717 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:25,717 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:26,641 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:26,717 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:27,717 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:28,718 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:29,717 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:30,720 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:31,641 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:31,721 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:32,718 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:33,718 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:34,617 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:26:34,617 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:26:34,718 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:35,719 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:36,719 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:36,731 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:37,268 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/config.yaml +2025-01-21 22:26:37,726 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:38,719 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:39,719 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:40,721 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:41,719 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:42,642 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:42,720 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:43,724 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:44,617 DEBUG SenderThread:2665561 [sender.py:send():379] send: exit +2025-01-21 22:26:44,617 INFO SenderThread:2665561 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:26:44,618 INFO SenderThread:2665561 [sender.py:send_exit():588] handling runtime: 265 +2025-01-21 22:26:44,619 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:44,619 INFO SenderThread:2665561 [sender.py:send_exit():594] send defer +2025-01-21 22:26:44,619 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:44,619 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:26:44,619 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:44,619 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:26:44,619 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:26:44,619 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:44,619 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:26:44,619 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:44,619 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:26:44,620 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:26:44,620 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:44,620 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:26:44,620 INFO HandlerThread:2665561 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:26:44,620 INFO HandlerThread:2665561 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:26:44,620 INFO HandlerThread:2665561 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:26:44,620 DEBUG SystemMonitor:2665561 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:26:44,621 DEBUG SystemMonitor:2665561 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:26:44,672 INFO HandlerThread:2665561 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:26:44,672 INFO HandlerThread:2665561 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:26:44,672 INFO HandlerThread:2665561 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:26:44,672 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:44,673 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:26:44,673 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:26:44,673 DEBUG SenderThread:2665561 [sender.py:send():379] send: stats +2025-01-21 22:26:44,673 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:44,673 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:26:44,673 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:44,673 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:26:44,673 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:26:44,673 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:44,673 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:26:45,271 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:26:46,272 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:51,008 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:26:51,008 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:51,009 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:26:51,010 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:51,010 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:26:51,010 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:26:51,010 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:26:51,010 DEBUG SenderThread:2665561 [sender.py:send():379] send: history +2025-01-21 22:26:51,010 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:26:51,010 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:51,011 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:26:51,011 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:51,011 DEBUG SenderThread:2665561 [sender.py:send():379] send: summary +2025-01-21 22:26:51,012 INFO SenderThread:2665561 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:26:51,012 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:51,012 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:26:51,012 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:26:51,012 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:51,012 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:26:51,012 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:51,012 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:26:51,012 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:26:51,012 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:26:51,013 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:51,013 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:26:51,013 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:51,013 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:26:51,274 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:26:51,618 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:26:52,221 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:26:52,221 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:26:52,221 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:52,221 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:26:52,221 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:52,221 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:26:52,221 INFO SenderThread:2665561 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:26:52,224 INFO SenderThread:2665561 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:26:52,226 INFO SenderThread:2665561 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:26:52,241 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:26:52,241 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:52,241 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:26:52,241 DEBUG SenderThread:2665561 [sender.py:send():379] send: artifact +2025-01-21 22:26:52,274 INFO Thread-12 :2665561 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:52,443 INFO SenderThread:2665561 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:26:52,443 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:52,443 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:26:52,443 INFO SenderThread:2665561 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:26:52,619 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:26:53,275 INFO SenderThread:2665561 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files +2025-01-21 22:26:53,275 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/config.yaml config.yaml +2025-01-21 22:26:53,275 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json wandb-summary.json +2025-01-21 22:26:53,275 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:26:53,275 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/requirements.txt requirements.txt +2025-01-21 22:26:53,275 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:26:53,276 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log output.log +2025-01-21 22:26:53,278 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/diff.patch diff.patch +2025-01-21 22:26:53,279 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:26:53,279 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/1_eval_1_e6e0be02eedcefa21d59.mp4 media/videos/eval/1_eval_1_e6e0be02eedcefa21d59.mp4 +2025-01-21 22:26:53,279 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/0_eval_0_6d4609e89d3ba1e1c763.mp4 media/videos/eval/0_eval_0_6d4609e89d3ba1e1c763.mp4 +2025-01-21 22:26:53,279 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/4_eval_4_92faad0d5acbe7e41d66.mp4 media/videos/eval/4_eval_4_92faad0d5acbe7e41d66.mp4 +2025-01-21 22:26:53,279 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/3_eval_3_56c00d964e4ff84c9625.mp4 media/videos/eval/3_eval_3_56c00d964e4ff84c9625.mp4 +2025-01-21 22:26:53,279 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/videos/eval/2_eval_2_941ccc9a5f52aef980c0.mp4 media/videos/eval/2_eval_2_941ccc9a5f52aef980c0.mp4 +2025-01-21 22:26:53,283 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/table/eval/success_rate_ci_5_e4f3d858401fb0f04765.table.json media/table/eval/success_rate_ci_5_e4f3d858401fb0f04765.table.json +2025-01-21 22:26:53,283 INFO SenderThread:2665561 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/media/table/eval/total_success_rate_ci_6_06392b86e92c89b6b251.table.json media/table/eval/total_success_rate_ci_6_06392b86e92c89b6b251.table.json +2025-01-21 22:26:53,284 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:26:53,284 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:26:53,284 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:53,284 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:26:53,284 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:53,284 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:26:53,284 INFO SenderThread:2665561 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:26:53,468 INFO wandb-upload_2:2665561 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json +2025-01-21 22:26:53,484 INFO wandb-upload_1:2665561 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/config.yaml +2025-01-21 22:26:53,521 INFO wandb-upload_0:2665561 [upload_job.py:push():130] Uploaded file /tmp/tmp417lnwgiwandb/3tiohtqq-events.out.tfevents.1737516139.leonmkim-ROG-Strix-G15CS-G15CS.2665365.0 +2025-01-21 22:26:53,552 INFO wandb-upload_4:2665561 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/output.log +2025-01-21 22:26:53,554 INFO wandb-upload_3:2665561 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/requirements.txt +2025-01-21 22:26:53,620 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:26:53,620 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:26:53,754 INFO Thread-11 (_thread_body):2665561 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:26:53,754 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:53,754 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:26:53,755 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:53,755 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:26:53,755 INFO SenderThread:2665561 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:26:53,756 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:26:53,756 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:53,756 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:26:53,756 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:53,756 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:26:53,756 INFO SenderThread:2665561 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:26:54,055 INFO SenderThread:2665561 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:26:54,055 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:26:54,055 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:54,055 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:26:54,055 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:54,055 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:26:54,055 INFO SenderThread:2665561 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:26:54,055 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:26:54,056 INFO HandlerThread:2665561 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:26:54,056 DEBUG SenderThread:2665561 [sender.py:send():379] send: final +2025-01-21 22:26:54,056 DEBUG SenderThread:2665561 [sender.py:send():379] send: footer +2025-01-21 22:26:54,056 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: defer +2025-01-21 22:26:54,056 INFO SenderThread:2665561 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:26:54,056 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:26:54,056 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:26:54,057 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:26:54,057 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:26:54,058 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:26:54,059 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:26:54,059 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:26:54,059 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:26:54,121 DEBUG SenderThread:2665561 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:26:54,121 INFO MainThread:2665561 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:26:54,122 INFO MainThread:2665561 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:26:54,122 INFO MainThread:2665561 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:26:54,122 DEBUG HandlerThread:2665561 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:26:54,122 INFO HandlerThread:2665561 [handler.py:finish():882] shutting down handler +2025-01-21 22:26:55,059 INFO WriterThread:2665561 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/run-1rhki7g4.wandb +2025-01-21 22:26:55,121 INFO SenderThread:2665561 [sender.py:finish():1615] shutting down sender +2025-01-21 22:26:55,121 INFO SenderThread:2665561 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:26:55,121 INFO SenderThread:2665561 [file_pusher.py:join():175] waiting for file pusher diff --git a/1rhki7g4/wandb/debug.log b/1rhki7g4/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5951cc31c4eb914a5faa996cb7dcd848f692d660 --- /dev/null +++ b/1rhki7g4/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 22:22:18,903 INFO MainThread:2665365 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 22:22:18,903 INFO MainThread:2665365 [wandb_setup.py:_flush():76] Configure stats pid to 2665365 +2025-01-21 22:22:18,903 INFO MainThread:2665365 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 22:22:18,903 INFO MainThread:2665365 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/settings +2025-01-21 22:22:18,903 INFO MainThread:2665365 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 22:22:18,903 INFO MainThread:2665365 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 22:22:18,903 INFO MainThread:2665365 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 22:22:18,903 INFO MainThread:2665365 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 22:22:18,903 INFO MainThread:2665365 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/logs/debug.log +2025-01-21 22:22:18,904 INFO MainThread:2665365 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/logs/debug-internal.log +2025-01-21 22:22:18,904 INFO MainThread:2665365 [wandb_init.py:init():569] calling init triggers +2025-01-21 22:22:18,904 INFO MainThread:2665365 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_2', 'wandb_notes': '4295_2_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4'} +2025-01-21 22:22:18,904 INFO MainThread:2665365 [wandb_init.py:init():619] starting backend +2025-01-21 22:22:18,904 INFO MainThread:2665365 [wandb_init.py:init():623] setting up manager +2025-01-21 22:22:18,908 INFO MainThread:2665365 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 22:22:18,909 INFO MainThread:2665365 [wandb_init.py:init():631] backend started and connected +2025-01-21 22:22:18,923 INFO MainThread:2665365 [wandb_init.py:init():720] updated telemetry +2025-01-21 22:22:18,930 INFO MainThread:2665365 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 22:22:19,195 INFO MainThread:2665365 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 22:22:19,263 INFO MainThread:2665365 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 22:22:19,263 INFO MainThread:2665365 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 22:22:19,613 INFO MainThread:2665365 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 22:22:19,613 INFO MainThread:2665365 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 22:22:19,613 INFO MainThread:2665365 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 22:22:19,613 INFO MainThread:2665365 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 22:22:19,615 INFO MainThread:2665365 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 22:22:19,616 INFO MainThread:2665365 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4/tb, True +2025-01-21 22:22:29,135 INFO MainThread:2665365 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:26:55,123 WARNING MsgRouterThr:2665365 [router.py:message_loop():77] message_loop has been closed diff --git a/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/config.yaml b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5446f75f14c2e4e3661291ffac96a0123c6f6277 --- /dev/null +++ b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 2 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_2' +wandb_notes: + desc: null + value: 4295_2_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 2 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/1rhki7g4 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737516138 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/diff.patch b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..d3a5568bb3f03cd705321e234cb5e34a755166b7 --- /dev/null +++ b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/diff.patch @@ -0,0 +1,25 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c37123f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -144,9 +144,9 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' ++# wandb_run_id: '4295_0' + # wandb_run_id: '4295_1' # seed 1 +-# wandb_run_id: '4295_2' # seed 2 ++wandb_run_id: '4295_2' # seed 2 + + # RGBD+mask+act history + # wandb_run_id: '1009_1' # seed 1 diff --git a/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/requirements.txt b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7f583e67944b1addbff23f72746a0a7e07b958ce --- /dev/null +++ b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "6d4609e89d3ba1e1c763d7eb0ec112a8c34479fbc6b28c43c867a6501ad50990", "size": 1562624, "path": "media/videos/eval/0_eval_0_6d4609e89d3ba1e1c763.mp4"}, "global_step": 535, "_timestamp": 1737516382.809059, "_runtime": 243.89955282211304, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "e6e0be02eedcefa21d59a5b50479ceefd38236aa85cb872ea490559629138b67", "size": 1512993, "path": "media/videos/eval/1_eval_1_e6e0be02eedcefa21d59.mp4"}, "eval/num_success": 2.0, "episode": 20.0, "eval/success_rate": 0.4000000059604645, "eval/success_rate_lb_0.01": 0.05313029885292053, "eval/success_rate_ub_0.01": 0.8694285750389099, "eval/success_rate_lb_0.025": 0.07380302250385284, "eval/success_rate_ub_0.025": 0.8284013867378235, "eval/success_rate_lb_0.05": 0.14315150678157806, "eval/success_rate_ub_0.05": 0.7544504404067993, "eval/success_rate_lb_0.1": 0.12242479622364044, "eval/success_rate_ub_0.1": 0.6598483920097351, "eval/2_eval": {"_type": "video-file", "sha256": "941ccc9a5f52aef980c0fa54fba565d745079da7a8e524f2dca8c0f882ee6bee", "size": 1505327, "path": "media/videos/eval/2_eval_2_941ccc9a5f52aef980c0.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "56c00d964e4ff84c96255a775a3ecd34a00f156a18f72e4b1d9942ba4eeaee52", "size": 853060, "path": "media/videos/eval/3_eval_3_56c00d964e4ff84c9625.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "92faad0d5acbe7e41d66e6f8ad175a9379c31c73d7d97f545eef706643b2d096", "size": 1012519, "path": "media/videos/eval/4_eval_4_92faad0d5acbe7e41d66.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e4f3d858401fb0f04765ec821f483d92f21a61c1e722e35130858291d0e8c5aa", "size": 401, "artifact_path": "wandb-client-artifact://feg3rrgcfi0epq9ve87yln5gwxc56ay2bfx6deet5viqvav51g8hewt7t515nwn1mge5sjucnylxq91pu1ck3w2okjxvmmzhucs1lo8s6lrihwh5h0ez1q4i9vja9dv7/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://y5edxpgwppj0ykwbwxdu9do0ybzjehi6hjhcwa0tq0c6hhckbmlcmkhsev1y6id5ccoajttgu31j5gn2u3zf3c6wxmgis3zzvrcf7bsz5niamzse0d5rxq65jncwh8au:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e4f3d858401fb0f04765.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 10, "total_num_episodes": 20, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.2775932848898335, "total_success_rate_ub_0.01": 0.7460466026767734, "total_success_rate_lb_0.025": 0.3081446737434721, "total_success_rate_ub_0.025": 0.7174153923553511, "total_success_rate_lb_0.05": 0.3230885789152155, "total_success_rate_ub_0.05": 0.6829789429537072, "total_success_rate_lb_0.1": 0.3550793528846665, "total_success_rate_ub_0.1": 0.6282084360462309, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "06392b86e92c89b6b2511b70de4025e8903ee925f5a82ee4ff74b34fadc2d2fc", "size": 505, "artifact_path": "wandb-client-artifact://jum4zax7tg2ljaw74tfwkty72kgw78g0oejj6qdo43zd30fnf03cu3u6xlcb10mqu7s6g6cfr8udzxlne7n4tq7es6a9vur9h89pwl3wd3xn1cs4uhg5dwudax8qq2kw/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://dknw7k20i7sj2krmeh5385rwqomrlzdu1wtkj098jqprj9vkpm4zy2wd9aelmfzzhz58irkwni9z6fwlrxp9zadez2uzqup9cvixjtb08k4r67ecpsl3oram5bqqcz1q:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_06392b86e92c89b6b251.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 265}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.2775932848453522, "eval/total_success_rate_ub_0.01": 0.7460466027259827, "eval/total_success_rate_lb_0.025": 0.3081446886062622, "eval/total_success_rate_ub_0.025": 0.7174153923988342, "eval/total_success_rate_lb_0.05": 0.3230885863304138, "eval/total_success_rate_ub_0.05": 0.6829789280891418, "eval/total_success_rate_lb_0.1": 0.3550793528556824, "eval/total_success_rate_ub_0.1": 0.6282084584236145} \ No newline at end of file diff --git a/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/run-1rhki7g4.wandb b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/run-1rhki7g4.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ad57863407d786f7bb5892d8c99bdc558934e441 Binary files /dev/null and b/1rhki7g4/wandb/run-20250121_222218-1rhki7g4/run-1rhki7g4.wandb differ diff --git a/7dl7quog/.hydra/config.yaml b/7dl7quog/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..836895e41a10194087484706499cf57e8cf16e48 --- /dev/null +++ b/7dl7quog/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/7dl7quog/.hydra/hydra.yaml b/7dl7quog/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1efba72009fcacfe329e6c4eb2c17693cb089326 --- /dev/null +++ b/7dl7quog/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/7dl7quog + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/7dl7quog/.hydra/overrides.yaml b/7dl7quog/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/7dl7quog/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/7dl7quog/eval_policy.log b/7dl7quog/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..92d50fcd45c5844b8e18a5af0b44e426b7adb8a7 --- /dev/null +++ b/7dl7quog/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 22:22:10,667][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 22:22:10,671][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 22:22:14,626][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/aq8qih16/.hydra/config.yaml b/aq8qih16/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..836895e41a10194087484706499cf57e8cf16e48 --- /dev/null +++ b/aq8qih16/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/aq8qih16/.hydra/hydra.yaml b/aq8qih16/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a176dbcdd6621e20d85a3799b11ffce1c394e55 --- /dev/null +++ b/aq8qih16/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/aq8qih16/.hydra/overrides.yaml b/aq8qih16/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/aq8qih16/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/aq8qih16/episode_rosbags/aligned_depth_to_color_K.npy b/aq8qih16/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/aq8qih16/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/aq8qih16/episode_rosbags/cam_tf_world.npy b/aq8qih16/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/aq8qih16/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/aq8qih16/episode_rosbags/color_K.npy b/aq8qih16/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/aq8qih16/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/aq8qih16/episode_rosbags/depth_K.npy b/aq8qih16/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/aq8qih16/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/aq8qih16/eval_robot.log b/aq8qih16/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..b7e931ddb95e9d3df7302033df36a9f6a771ce0a --- /dev/null +++ b/aq8qih16/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:56:21,687][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:56:21,690][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/aq8qih16/eval_video/0_eval.mp4 b/aq8qih16/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..411399f8d818a6e2cc2d7c5586f50aa64e1f98a4 --- /dev/null +++ b/aq8qih16/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d41b9de2cc77d8823f41c9ce392d45db97f4cdbee28c695d5b9e5066fc0fa2d9 +size 1478431 diff --git a/aq8qih16/eval_video/1_eval.mp4 b/aq8qih16/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e410a101edcda5781ceaf6b62686257042efa560 --- /dev/null +++ b/aq8qih16/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e52801f26f69cbfb59094e0fa350ed50d7610612ebcd007d1a5bf1aa88866e9 +size 1139082 diff --git a/aq8qih16/eval_video/2_eval.mp4 b/aq8qih16/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..236dfe37c16112502d90c3b637fc151d35d93822 --- /dev/null +++ b/aq8qih16/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee9b6581bc6fe6d9539b8782cff664c1f598601591f030f2fafa124c9dd6fd58 +size 996088 diff --git a/aq8qih16/eval_video/3_eval.mp4 b/aq8qih16/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cdd070c093c0b7b6dc7d59481a098fe9b31f2a8d --- /dev/null +++ b/aq8qih16/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ef491d5f6f41b1a481b9c2c33345629f3511d30066ff4033a67eef3efc2488f +size 1015076 diff --git a/aq8qih16/eval_video/4_eval.mp4 b/aq8qih16/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f29c979e2923df90e84ac616aa848ceb9a031bf4 --- /dev/null +++ b/aq8qih16/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bc7e953a75c0b8f46760b9741d9f78fe2a0aef7faa8efab5ca281f25b7fad61 +size 1404597 diff --git a/aq8qih16/tb/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 b/aq8qih16/tb/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 new file mode 100644 index 0000000000000000000000000000000000000000..e3505f310bf7efd7a62680b6a06b702c0a7cdcd5 --- /dev/null +++ b/aq8qih16/tb/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72d57f09a020deb88705dceae5177ec3f645d6e9707ae8c0ca4fa5695075dcbb +size 6539 diff --git a/aq8qih16/wandb/debug-internal.log b/aq8qih16/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2a54fcd290e5b92deb558bfbd29388b0300cdb6d --- /dev/null +++ b/aq8qih16/wandb/debug-internal.log @@ -0,0 +1,746 @@ +2025-01-21 21:56:27,645 INFO StreamThr :2658709 [internal.py:wandb_internal():85] W&B internal server running at pid: 2658709, started at: 2025-01-21 21:56:27.645113 +2025-01-21 21:56:27,646 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:56:27,656 INFO WriterThread:2658709 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/run-aq8qih16.wandb +2025-01-21 21:56:27,657 DEBUG SenderThread:2658709 [sender.py:send():379] send: header +2025-01-21 21:56:27,665 DEBUG SenderThread:2658709 [sender.py:send():379] send: run +2025-01-21 21:56:27,901 INFO SenderThread:2658709 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files +2025-01-21 21:56:27,901 INFO SenderThread:2658709 [sender.py:_start_run_threads():1188] run started: aq8qih16 with start time 1737514587.645315 +2025-01-21 21:56:27,917 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:56:27,917 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:56:27,970 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:56:27,974 DEBUG HandlerThread:2658709 [system_info.py:__init__():26] System info init +2025-01-21 21:56:27,975 DEBUG HandlerThread:2658709 [system_info.py:__init__():41] System info init done +2025-01-21 21:56:27,975 INFO HandlerThread:2658709 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:56:27,975 INFO SystemMonitor:2658709 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:56:27,975 INFO HandlerThread:2658709 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:56:27,976 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:56:27,976 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:56:27,977 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:56:27,977 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:56:27,978 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:56:27,987 DEBUG HandlerThread:2658709 [system_info.py:probe():152] Probing system +2025-01-21 21:56:27,989 DEBUG HandlerThread:2658709 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:56:27,993 DEBUG HandlerThread:2658709 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:56:27,993 DEBUG HandlerThread:2658709 [system_info.py:probe():200] Probing system done +2025-01-21 21:56:27,993 DEBUG HandlerThread:2658709 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:56:27.987764', 'startedAt': '2025-01-22T02:56:27.632967', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3745.9728750000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.83, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.985, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 428.19886779785156}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:56:27,993 INFO HandlerThread:2658709 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:56:27,993 INFO HandlerThread:2658709 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:56:27,993 DEBUG HandlerThread:2658709 [system_info.py:_save_code():44] Saving code +2025-01-21 21:56:27,995 DEBUG HandlerThread:2658709 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:56:27,995 DEBUG HandlerThread:2658709 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:56:28,057 DEBUG HandlerThread:2658709 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:56:28,057 INFO HandlerThread:2658709 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:56:28,060 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:56:28,060 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:56:28,060 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:56:28,061 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:56:28,297 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/6lfg37oo-wandb-metadata.json +2025-01-21 21:56:28,303 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:56:28,303 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:56:28,306 DEBUG SenderThread:2658709 [sender.py:send():379] send: telemetry +2025-01-21 21:56:28,353 INFO wandb-upload_2:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/zsq0kvhv-diff.patch +2025-01-21 21:56:28,353 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:56:28,353 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:28,354 INFO HandlerThread:2658709 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/tb" + save: true +} +control { + relay_id: "b4a5101f82b94a7399ab6e78a37f9fea" +} +_info { + stream_id: "aq8qih16" +} + +2025-01-21 21:56:28,354 DEBUG HandlerThread:2658709 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:56:28,355 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:56:28,362 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/o68q7yps-code/FISH/eval_robot.py +2025-01-21 21:56:28,504 DEBUG SenderThread:2658709 [sender.py:send():379] send: tbrecord +2025-01-21 21:56:28,505 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:56:28,505 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 with policy live +2025-01-21 21:56:28,698 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/6e9difk8-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:56:28,903 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/requirements.txt +2025-01-21 21:56:28,903 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code/FISH/eval_robot.py +2025-01-21 21:56:28,903 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:28,903 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:56:28,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-metadata.json +2025-01-21 21:56:28,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/diff.patch +2025-01-21 21:56:28,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code +2025-01-21 21:56:28,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code/FISH +2025-01-21 21:56:29,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:30,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:30,902 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:31,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:32,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:33,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:33,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:34,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:35,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:36,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:36,691 DEBUG SenderThread:2658709 [sender.py:send():379] send: config +2025-01-21 21:56:37,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:38,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:38,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:38,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:39,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:40,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:41,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:42,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:42,906 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:43,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:56:43,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:56:43,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:43,405 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:44,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:45,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:46,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:46,907 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:47,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:48,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:49,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:49,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:50,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:51,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:52,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:52,908 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:53,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:54,309 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:54,909 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:55,245 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:55,309 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:56,309 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:56,910 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:57,309 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:58,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:56:58,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:56:58,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:59,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:00,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:00,453 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:00,911 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml +2025-01-21 21:57:01,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:02,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:03,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:04,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:04,912 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:05,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:05,649 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:06,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:07,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:08,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:09,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:10,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:11,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:11,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:12,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:12,914 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:13,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:57:13,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:57:13,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:14,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:15,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:16,312 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:16,389 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:17,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:18,312 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:19,312 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:20,312 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:21,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:22,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:22,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:23,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:24,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:25,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:26,314 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:27,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:27,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:27,978 DEBUG SystemMonitor:2658709 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:57:27,979 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:57:28,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:57:28,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:57:28,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:29,314 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:29,653 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:57:29,653 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 with policy now +2025-01-21 21:57:29,653 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:57:29,654 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:57:29,654 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:57:29,655 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:57:29,918 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 +2025-01-21 21:57:29,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:57:29,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos +2025-01-21 21:57:29,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 21:57:29,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media +2025-01-21 21:57:29,926 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/ne2t3wkp-media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 +2025-01-21 21:57:30,258 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/tz6rowed-media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 +2025-01-21 21:57:30,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:30,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:31,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:32,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:32,666 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:33,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:34,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:34,920 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:35,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:36,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:36,921 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:37,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:38,033 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:38,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:39,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:40,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:40,922 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:57:41,213 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/2fd02t53-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:57:41,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:42,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:42,923 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:43,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:57:43,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:57:43,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:43,373 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:44,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:45,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:46,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:46,924 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:47,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:48,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:49,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:49,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:50,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:51,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:52,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:53,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:54,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:55,225 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:55,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:56,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:56,927 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:57,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:57,979 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:57:58,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:57:58,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:57:58,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:58,928 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:59,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:00,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:00,385 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:01,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:02,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:03,319 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:04,321 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:04,930 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:05,320 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:06,178 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:06,320 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:07,320 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:08,321 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:08,932 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:09,321 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:09,792 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:58:09,792 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 with policy now +2025-01-21 21:58:09,793 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:58:09,793 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:58:09,793 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:58:09,794 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:58:09,932 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:58:09,932 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 +2025-01-21 21:58:09,933 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 21:58:10,062 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/tj1v9neo-media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 +2025-01-21 21:58:10,337 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:10,345 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/y6ppesb6-media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 +2025-01-21 21:58:10,933 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:11,325 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:11,811 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:11,933 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:12,329 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:13,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:58:13,307 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:58:13,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:14,328 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:14,934 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:15,327 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:16,325 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:16,935 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:17,093 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:17,325 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:18,326 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:19,326 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:19,759 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:58:19,760 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:58:19,771 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:58:19,936 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:58:19,936 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:58:20,326 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:20,936 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:21,326 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:22,327 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:22,509 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:22,937 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:23,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:24,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:24,937 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:25,331 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:26,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:27,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:27,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:27,979 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:58:28,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:58:28,305 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:58:28,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:29,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:30,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:31,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:32,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:33,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:33,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:34,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:35,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:36,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:37,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:38,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:38,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:39,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:40,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:41,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:42,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:43,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:58:43,305 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:58:43,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:43,402 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:44,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:45,338 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:45,837 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:58:45,838 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 with policy now +2025-01-21 21:58:45,842 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:58:45,842 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:58:45,843 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:58:45,843 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:58:45,945 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:58:45,946 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 +2025-01-21 21:58:45,946 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 21:58:46,095 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/nelnvo5q-media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 +2025-01-21 21:58:46,337 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:46,386 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/ei6lq6iz-media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 +2025-01-21 21:58:46,946 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:47,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:47,946 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:48,347 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:49,271 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:49,342 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:50,348 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:50,947 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:51,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:52,340 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:52,948 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:52,948 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:58:53,235 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/sxid0tpy-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:58:53,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:53,582 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:58:53,582 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:58:53,582 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:58:53,948 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:58:54,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:54,948 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:54,965 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:55,348 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:56,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:57,347 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:57,980 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:58:58,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:58:58,305 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:58:58,347 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:58,950 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:59,348 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:00,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:00,445 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:01,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:02,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:03,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:04,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:05,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:06,334 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:06,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:07,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:08,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:09,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:10,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:11,335 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:11,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:12,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:13,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:59:13,305 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:59:13,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:14,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:15,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:16,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:16,641 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:16,956 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:17,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:18,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:19,357 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:19,887 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:59:19,887 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 with policy now +2025-01-21 21:59:19,893 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:59:19,894 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:59:19,894 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:59:19,895 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:59:19,957 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:59:19,957 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 +2025-01-21 21:59:19,957 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 21:59:20,150 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/8ylus15t-media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 +2025-01-21 21:59:20,353 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:20,425 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/tflqghtb-media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 +2025-01-21 21:59:20,957 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:21,358 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:21,958 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:21,960 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:22,357 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:23,359 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:24,361 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:24,959 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:25,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:26,355 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:26,622 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:59:26,622 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:59:26,622 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:59:26,959 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:26,960 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:59:26,960 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:59:27,291 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:27,359 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:27,980 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:59:28,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:59:28,313 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:59:28,370 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:28,960 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:29,366 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:30,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:30,961 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:31,361 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:32,361 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:32,569 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:32,962 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:33,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:34,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:35,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:36,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:37,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:38,336 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:38,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:39,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:40,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:41,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:42,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:43,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:59:43,306 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:59:43,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:43,405 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:44,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:45,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:46,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:47,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:48,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:49,337 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:49,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:50,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:51,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:52,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:53,366 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:54,338 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:54,366 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:55,368 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:56,368 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:57,368 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:57,981 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:59:58,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:59:58,306 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:59:58,369 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:59,368 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:59,463 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:00,369 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:01,370 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:02,372 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:02,972 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:03,384 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:03,504 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 22:00:03,504 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 with policy now +2025-01-21 22:00:03,507 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:00:03,508 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:03,508 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:03,508 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:03,763 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/74h73bsl-media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 +2025-01-21 22:00:03,972 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:03,973 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 +2025-01-21 22:00:03,973 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 22:00:04,264 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/5v3wqnh9-media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 +2025-01-21 22:00:04,388 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:04,552 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:04,973 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:05,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:05,973 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:06,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:07,400 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:08,400 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:08,974 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:09,390 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:09,918 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:10,407 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:10,975 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:11,410 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:12,406 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:13,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:00:13,316 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:00:13,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:14,403 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:14,679 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:14,680 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:14,680 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:14,976 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:00:14,976 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:00:14,977 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:14,977 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:15,181 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:00:15,248 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/esa4nea3-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:15,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:15,468 INFO wandb-upload_0:2658709 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyitzqrk6 +2025-01-21 22:00:15,543 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:00:15,555 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:00:15,904 INFO SenderThread:2658709 [sender.py:send_request_log_artifact():1525] logged artifact run-aq8qih16-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcwODM5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMjM0OA==', 'latestArtifact': None}} +2025-01-21 22:00:15,904 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 22:00:15,904 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json with policy now +2025-01-21 22:00:15,905 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:15,905 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:15,905 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,905 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,906 DEBUG SenderThread:2658709 [sender.py:send():379] send: telemetry +2025-01-21 22:00:15,906 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,906 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,906 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,908 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,908 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,908 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,909 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,909 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,909 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,909 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,910 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,910 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,910 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,910 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,911 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,911 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,911 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,912 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,912 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,912 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,912 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,913 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,913 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval +2025-01-21 22:00:16,107 INFO wandb-upload_2:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/8reeflyc-media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json +2025-01-21 22:00:16,167 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/nh0voxvz-media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json +2025-01-21 22:00:16,310 INFO wandb-upload_1:2658709 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfy9mhbrg +2025-01-21 22:00:16,377 INFO wandb-upload_2:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/f8ynsx6d-media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json +2025-01-21 22:00:16,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:16,738 INFO SenderThread:2658709 [sender.py:send_request_log_artifact():1525] logged artifact run-aq8qih16-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcwODQwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMjM1MQ==', 'latestArtifact': None}} +2025-01-21 22:00:16,738 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 22:00:16,738 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json with policy now +2025-01-21 22:00:16,738 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:16,738 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:16,738 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:16,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:16,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:17,009 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/mgo68mow-media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json +2025-01-21 22:00:17,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:18,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:19,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:20,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:21,339 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:21,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:22,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:23,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:24,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:25,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:26,339 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:26,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:27,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:27,981 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 22:00:28,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:00:28,306 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:00:28,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:29,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:30,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:31,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:31,403 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:32,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:33,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:34,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:35,396 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:36,396 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:36,533 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:36,577 DEBUG SenderThread:2658709 [sender.py:send():379] send: exit +2025-01-21 22:00:36,577 INFO SenderThread:2658709 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:send_exit():588] handling runtime: 248 +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:send_exit():594] send defer +2025-01-21 22:00:36,578 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,578 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:00:36,578 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:00:36,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,579 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:00:36,579 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:36,579 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:00:36,579 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:00:36,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,579 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:00:36,579 INFO HandlerThread:2658709 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:00:36,579 DEBUG SystemMonitor:2658709 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:00:36,579 DEBUG SystemMonitor:2658709 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:00:36,579 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:00:36,580 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:00:36,632 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:00:36,632 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:00:36,632 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:00:36,633 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:36,633 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:00:36,633 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:00:36,633 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 22:00:36,633 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,633 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:00:36,633 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:36,633 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:00:36,634 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:00:36,634 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,634 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:00:36,986 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:38,987 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:42,686 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:42,687 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:42,687 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:00:42,688 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:42,688 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:00:42,688 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:00:42,688 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:42,688 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:42,688 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:42,688 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:00:42,688 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:42,689 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:42,690 DEBUG SenderThread:2658709 [sender.py:send():379] send: summary +2025-01-21 22:00:42,693 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:42,694 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:42,694 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:00:42,694 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:00:42,694 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:42,694 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:00:42,694 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:42,694 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:00:42,697 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:42,847 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:00:42,848 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:42,848 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:00:42,848 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:42,848 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:00:42,988 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:42,989 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml +2025-01-21 22:00:43,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:45,741 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:00:45,741 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:45,741 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:45,741 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:00:45,742 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:45,742 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:00:45,742 INFO SenderThread:2658709 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:00:45,744 INFO SenderThread:2658709 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:00:45,746 INFO SenderThread:2658709 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:00:45,760 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:00:45,760 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:45,760 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:00:45,760 DEBUG SenderThread:2658709 [sender.py:send():379] send: artifact +2025-01-21 22:00:45,990 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:46,003 INFO SenderThread:2658709 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:00:46,004 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:46,004 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:00:46,004 INFO SenderThread:2658709 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:00:46,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml config.yaml +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json wandb-summary.json +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/requirements.txt requirements.txt +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:46,993 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log output.log +2025-01-21 22:00:46,993 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/diff.patch diff.patch +2025-01-21 22:00:46,995 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:00:46,996 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 +2025-01-21 22:00:46,996 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:00:46,997 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:46,997 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:46,998 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:00:46,999 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:46,999 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:00:46,999 INFO SenderThread:2658709 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:00:47,179 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/requirements.txt +2025-01-21 22:00:47,184 INFO wandb-upload_2:2658709 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml +2025-01-21 22:00:47,220 INFO wandb-upload_4:2658709 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:47,253 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:47,297 INFO wandb-upload_3:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/6i9eht82-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:47,498 INFO Thread-11 (_thread_body):2658709 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:00:47,498 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:47,498 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:00:47,499 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:47,499 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:00:47,499 INFO SenderThread:2658709 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:00:47,500 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:00:47,500 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:47,500 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:00:47,500 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:47,500 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:00:47,500 INFO SenderThread:2658709 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:00:47,580 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:47,713 INFO SenderThread:2658709 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:00:47,713 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:00:47,713 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:47,713 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:47,713 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:00:47,713 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:47,713 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:00:47,713 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:00:47,713 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:47,713 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:00:47,714 DEBUG SenderThread:2658709 [sender.py:send():379] send: final +2025-01-21 22:00:47,714 DEBUG SenderThread:2658709 [sender.py:send():379] send: footer +2025-01-21 22:00:47,714 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:47,714 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:00:47,714 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:47,715 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:47,715 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:00:47,715 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:47,715 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:00:47,717 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:00:47,717 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:00:47,718 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:47,768 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:47,768 INFO MainThread:2658709 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:00:47,769 INFO MainThread:2658709 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:00:47,769 INFO MainThread:2658709 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:00:47,769 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:00:47,769 INFO HandlerThread:2658709 [handler.py:finish():882] shutting down handler +2025-01-21 22:00:48,715 INFO WriterThread:2658709 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/run-aq8qih16.wandb +2025-01-21 22:00:48,768 INFO SenderThread:2658709 [sender.py:finish():1615] shutting down sender +2025-01-21 22:00:48,768 INFO SenderThread:2658709 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:00:48,769 INFO SenderThread:2658709 [file_pusher.py:join():175] waiting for file pusher diff --git a/aq8qih16/wandb/debug.log b/aq8qih16/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..882acb80422dc50fffae91669710a24d21bae9c8 --- /dev/null +++ b/aq8qih16/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Configure stats pid to 2658535 +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/settings +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug.log +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug-internal.log +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_2', 'wandb_notes': '4295_2_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16'} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:init():619] starting backend +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:init():623] setting up manager +2025-01-21 21:56:27,644 INFO MainThread:2658535 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:56:27,645 INFO MainThread:2658535 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:56:27,656 INFO MainThread:2658535 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:56:27,663 INFO MainThread:2658535 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:56:27,917 INFO MainThread:2658535 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:56:27,964 INFO MainThread:2658535 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:56:27,964 INFO MainThread:2658535 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:56:28,304 INFO MainThread:2658535 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:56:28,304 INFO MainThread:2658535 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:56:28,305 INFO MainThread:2658535 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:56:28,305 INFO MainThread:2658535 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:56:28,307 INFO MainThread:2658535 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:56:28,307 INFO MainThread:2658535 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/tb, True +2025-01-21 21:56:36,690 INFO MainThread:2658535 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:00:48,770 WARNING MsgRouterThr:2658535 [router.py:message_loop():77] message_loop has been closed diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code/FISH/eval_robot.py b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6fdb0eaf8094c5c7043f4b7102c72593411cb38a --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 2 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_2' +wandb_notes: + desc: null + value: 4295_2_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 2 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737514587 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/diff.patch b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..d3a5568bb3f03cd705321e234cb5e34a755166b7 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/diff.patch @@ -0,0 +1,25 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c37123f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -144,9 +144,9 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' ++# wandb_run_id: '4295_0' + # wandb_run_id: '4295_1' # seed 1 +-# wandb_run_id: '4295_2' # seed 2 ++wandb_run_id: '4295_2' # seed 2 + + # RGBD+mask+act history + # wandb_run_id: '1009_1' # seed 1 diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json new file mode 100644 index 0000000000000000000000000000000000000000..237154e062fa3d2fbc0e6466672f7cb5dacc26a9 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.2, 0.0055151880776414695, 0.7395095228669846, 0.010103650489081106, 0.6807302236195546, 0.04013022789615792, 0.5821626335218136, 0.0248521492835481, 0.4661657214232404]]} \ No newline at end of file diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json new file mode 100644 index 0000000000000000000000000000000000000000..55b1c035b6a70fdefa339c0fbc833fd1d804b0a4 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[8, 15, 0.5333333333333333, 0.2787657976592981, 0.8038517832148339, 0.31289437416957744, 0.7751986383841387, 0.32704703513266653, 0.738412201356889, 0.36331460627666956, 0.6756605133062947]]} \ No newline at end of file diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..411399f8d818a6e2cc2d7c5586f50aa64e1f98a4 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d41b9de2cc77d8823f41c9ce392d45db97f4cdbee28c695d5b9e5066fc0fa2d9 +size 1478431 diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e410a101edcda5781ceaf6b62686257042efa560 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e52801f26f69cbfb59094e0fa350ed50d7610612ebcd007d1a5bf1aa88866e9 +size 1139082 diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..236dfe37c16112502d90c3b637fc151d35d93822 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ee9b6581bc6fe6d9539b8782cff664c1f598601591f030f2fafa124c9dd6fd58 +size 996088 diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cdd070c093c0b7b6dc7d59481a098fe9b31f2a8d --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ef491d5f6f41b1a481b9c2c33345629f3511d30066ff4033a67eef3efc2488f +size 1015076 diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..f29c979e2923df90e84ac616aa848ceb9a031bf4 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5bc7e953a75c0b8f46760b9741d9f78fe2a0aef7faa8efab5ca281f25b7fad61 +size 1404597 diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..27b89e382d46a7bd9e7708c52663bbc05d7a42c4 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log @@ -0,0 +1,291 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737514596.691561]: resetting environment +[INFO] [1737514596.699027]: cleared current plan +[INFO] [1737514596.700395]: moving to home +[INFO] [1737514597.702531]: reached home +[INFO] [1737514597.703768]: reset action history +[INFO] [1737514600.413148]: environment reset +Starting episode 0 +[INFO] [1737514600.417071]: resetting environment +[INFO] [1737514600.425348]: cleared current plan +[INFO] [1737514600.426590]: moving to home +[INFO] [1737514601.428677]: reached home +[INFO] [1737514601.429937]: reset action history +[INFO] [1737514604.139755]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737514610.243278]: resetting environment +[INFO] [1737514610.247497]: cleared current plan +[INFO] [1737514610.248347]: moving to home +[INFO] [1737514611.251252]: reached home +[INFO] [1737514611.252508]: reset action history +[INFO] [1737514613.961512]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/episode_rosbags/episode_0_2025-01-21-21-56-53.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2658535.log +started roslaunch server http://158.130.50.37:37103/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2658816] +process[print_text-2]: started with pid [2658817] +process[rosbag_record-3]: started with pid [2658841] +started bagging! +For topic gripper_width: timestamp difference is -110124898 for nearest: 1737514623412475753 - target: 1737514623522600651 at idx 118 +For topic gripper_width: timestamp difference is 104995647 for nearest: 1737514631879097381 - target: 1737514631774101734 at idx 227 +[INFO] [1737514648.649560]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1737514649.432774]: Stored episode 1. +[INFO] [1737514649.433055]: Saving video... +[INFO] [1737514649.620873]: Video saved! +Starting episode 1 +[INFO] [1737514649.654856]: resetting environment +[INFO] [1737514649.665393]: cleared current plan +[INFO] [1737514649.665646]: moving to home +[INFO] [1737514653.268612]: reached home +[INFO] [1737514653.269476]: reset action history +[INFO] [1737514655.985281]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514660.919071]: resetting environment +[INFO] [1737514660.922799]: cleared current plan +[INFO] [1737514660.923004]: moving to home +[INFO] [1737514661.924446]: reached home +[INFO] [1737514661.924851]: reset action history +[INFO] [1737514664.631682]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/episode_rosbags/episode_1_2025-01-21-21-57-44.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2658535.log +started roslaunch server http://158.130.50.37:33751/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2658966] +process[print_text-5]: started with pid [2658967] +process[rosbag_record-6]: started with pid [2658991] +started bagging! +For topic gripper_width: timestamp difference is 106593307 for nearest: 1737514674179100450 - target: 1737514674072507143 at idx 131 +For topic gripper_width: timestamp difference is 103005633 for nearest: 1737514676479166970 - target: 1737514676376161337 at idx 158 +For topic gripper_width: timestamp difference is 107633475 for nearest: 1737514682212495927 - target: 1737514682104862452 at idx 229 +For topic gripper_width: timestamp difference is 110766128 for nearest: 1737514686179140285 - target: 1737514686068374157 at idx 249 +preempting episode +[INFO] [1737514688.385825]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737514689.151948]: Stored episode 2. +[INFO] [1737514689.152710]: Saving video... +[INFO] [1737514689.758224]: Video saved! +Starting episode 2 +[INFO] [1737514689.797635]: resetting environment +[INFO] [1737514689.810071]: cleared current plan +[INFO] [1737514689.810295]: moving to home +[WARN] [1737514689.569529]: Plan exhausted +[WARN] [1737514689.603680]: Plan exhausted +[WARN] [1737514689.643940]: Plan exhausted +[WARN] [1737514689.682726]: Plan exhausted +[WARN] [1737514689.723833]: Plan exhausted +[WARN] [1737514689.762747]: Plan exhausted +[WARN] [1737514689.802912]: Plan exhausted +[INFO] [1737514693.313321]: reached home +[INFO] [1737514693.314041]: reset action history +[INFO] [1737514696.046713]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514699.453717]: resetting environment +[INFO] [1737514699.464025]: cleared current plan +[INFO] [1737514699.464673]: moving to home +[INFO] [1737514700.465255]: reached home +[INFO] [1737514700.466124]: reset action history +[INFO] [1737514703.176436]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/episode_rosbags/episode_2_2025-01-21-21-58-23.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2658535.log +started roslaunch server http://158.130.50.37:39825/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2659131] +process[print_text-8]: started with pid [2659132] +process[rosbag_record-9]: started with pid [2659133] +started bagging! +preempting episode +[INFO] [1737514724.738794]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1737514725.479513]: Stored episode 3. +[INFO] [1737514725.482895]: Saving video... +[INFO] [1737514725.797913]: Video saved! +Starting episode 3 +[INFO] [1737514725.848090]: resetting environment +[INFO] [1737514725.864290]: cleared current plan +[INFO] [1737514725.864522]: moving to home +[WARN] [1737514725.856519]: Plan exhausted +[INFO] [1737514729.266786]: reached home +[INFO] [1737514729.278136]: reset action history +[INFO] [1737514732.065395]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514732.893853]: resetting environment +[INFO] [1737514732.918730]: cleared current plan +[INFO] [1737514732.919291]: moving to home +[INFO] [1737514733.924172]: reached home +[INFO] [1737514733.925422]: reset action history +[INFO] [1737514736.703683]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/episode_rosbags/episode_3_2025-01-21-21-58-56.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2658535.log +started roslaunch server http://158.130.50.37:39607/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2659249] +process[print_text-11]: started with pid [2659250] +process[rosbag_record-12]: started with pid [2659274] +started bagging! +For topic gripper_width: timestamp difference is -105606791 for nearest: 1737514755412481311 - target: 1737514755518088102 at idx 249 +preempting episode +[INFO] [1737514758.329752]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1737514759.183883]: Stored episode 4. +[INFO] [1737514759.197186]: Saving video... +[INFO] [1737514759.846453]: Video saved! +Starting episode 4 +[INFO] [1737514759.911809]: resetting environment +[INFO] [1737514759.953629]: cleared current plan +[INFO] [1737514759.958289]: moving to home +[WARN] [1737514759.578898]: Plan exhausted +[WARN] [1737514759.616068]: Plan exhausted +[WARN] [1737514759.655707]: Plan exhausted +[WARN] [1737514759.697895]: Plan exhausted +[WARN] [1737514759.743804]: Plan exhausted +[WARN] [1737514759.786479]: Plan exhausted +[WARN] [1737514759.815648]: Plan exhausted +[WARN] [1737514759.856194]: Plan exhausted +[WARN] [1737514759.895733]: Plan exhausted +[WARN] [1737514759.936064]: Plan exhausted +[INFO] [1737514763.366534]: reached home +[INFO] [1737514763.377586]: reset action history +[INFO] [1737514766.173014]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514766.255645]: resetting environment +[INFO] [1737514766.276761]: cleared current plan +[INFO] [1737514766.277018]: moving to home +[INFO] [1737514767.288630]: reached home +[INFO] [1737514767.292424]: reset action history +[INFO] [1737514770.135118]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/episode_rosbags/episode_4_2025-01-21-21-59-30.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2658535.log +started roslaunch server http://158.130.50.37:36367/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2659411] +process[print_text-14]: started with pid [2659412] +process[rosbag_record-15]: started with pid [2659413] +started bagging! +preempting episode +[INFO] [1737514802.147038]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[print_text-14] killing on exit +[INFO] [1737514802.902741]: Stored episode 5. +[INFO] [1737514802.906340]: Saving video... +[INFO] [1737514803.448148]: Video saved! +[INFO] [1737514803.508600]: resetting environment +[INFO] [1737514803.551344]: cleared current plan +[INFO] [1737514803.551591]: moving to home +[WARN] [1737514803.456230]: Plan exhausted +[WARN] [1737514803.495633]: Plan exhausted +[WARN] [1737514803.535896]: Plan exhausted +[INFO] [1737514807.053224]: reached home +[INFO] [1737514807.064935]: reset action history +[INFO] [1737514809.915953]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/requirements.txt b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-metadata.json b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..20abcd7f1f1a7e41b2794ac945343cb8e5127166 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:56:27.987764", + "startedAt": "2025-01-22T02:56:27.632967", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3745.9728750000004, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.83, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.985, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 428.19886779785156 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..9bff569880c7db4f95822113be614695a5b13f1c --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "d41b9de2cc77d8823f41c9ce392d45db97f4cdbee28c695d5b9e5066fc0fa2d9", "size": 1478431, "path": "media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4"}, "global_step": 508, "_timestamp": 1737514815.465184, "_runtime": 227.81986904144287, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "5e52801f26f69cbfb59094e0fa350ed50d7610612ebcd007d1a5bf1aa88866e9", "size": 1139082, "path": "media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4"}, "eval/num_success": 1.0, "episode": 15.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.005515187978744507, "eval/success_rate_ub_0.01": 0.7395095229148865, "eval/success_rate_lb_0.025": 0.010103650391101837, "eval/success_rate_ub_0.025": 0.6807302236557007, "eval/success_rate_lb_0.05": 0.04013022780418396, "eval/success_rate_ub_0.05": 0.582162618637085, "eval/success_rate_lb_0.1": 0.024852149188518524, "eval/success_rate_ub_0.1": 0.4661657214164734, "eval/2_eval": {"_type": "video-file", "sha256": "ee9b6581bc6fe6d9539b8782cff664c1f598601591f030f2fafa124c9dd6fd58", "size": 996088, "path": "media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "9ef491d5f6f41b1a481b9c2c33345629f3511d30066ff4033a67eef3efc2488f", "size": 1015076, "path": "media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "5bc7e953a75c0b8f46760b9741d9f78fe2a0aef7faa8efab5ca281f25b7fad61", "size": 1404597, "path": "media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "50e92e293996504485a1fa27b6ed21270fccc87ab31cf0d8d84a1873ee6cbb4a", "size": 404, "artifact_path": "wandb-client-artifact://bucj6c7v8lqnkziegos05hx2dzq1a31dfo3xxgpq8906s2eg0nq4o9i3cubm03tc5d2te6vwidd7n0bim79azicexerdt1vy3bqpjuog56f2qbo742d1so9dr1it57vb/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://nbdy8c5hrjm63ihjpugwnu1pbp8809kbojawemld6meudm4l9eoiztc4z8qbrl11xht2hx7glox5fce2hqqo7gg85q9yvy1vgy68trbvp261f77l5h5bbdarr6r73kxx:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 8, "total_num_episodes": 15, "total_success_rate": 0.5333333333333333, "total_success_rate_lb_0.01": 0.2787657976592981, "total_success_rate_ub_0.01": 0.8038517832148339, "total_success_rate_lb_0.025": 0.31289437416957744, "total_success_rate_ub_0.025": 0.7751986383841387, "total_success_rate_lb_0.05": 0.32704703513266653, "total_success_rate_ub_0.05": 0.738412201356889, "total_success_rate_lb_0.1": 0.36331460627666956, "total_success_rate_ub_0.1": 0.6756605133062947, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "5f51e5dd50014acd45f6a5e57829539146a0c816a50558ec6aa11a543142f5db", "size": 521, "artifact_path": "wandb-client-artifact://qm35lqd09exxa9axx1mg1s1sf0ah50vp427a245v9uty6kznlqcgpkxcnvfd1oxsfbjt641t7fkcqqf4mt067ih4tjheyimyezg1rg9xxwyg4vy0kpdb41axqlc1534g/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://5yxcapwh0nqgmooeh1lyg0atmgvft4zl5dltdgs78obq4bczh0mevoses3lf5pyvkh0f74b5xdaeoagwpgrferh63vl6yrt91lyg00ln1gsv2jmq69dz5pznx2daav1k:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 248}, "eval/total_success_rate": 0.5333333611488342, "eval/total_success_rate_lb_0.01": 0.27876579761505127, "eval/total_success_rate_ub_0.01": 0.8038517832756042, "eval/total_success_rate_lb_0.025": 0.31289437413215637, "eval/total_success_rate_ub_0.025": 0.7751986384391785, "eval/total_success_rate_lb_0.05": 0.32704704999923706, "eval/total_success_rate_ub_0.05": 0.7384122014045715, "eval/total_success_rate_lb_0.1": 0.36331459879875183, "eval/total_success_rate_ub_0.1": 0.6756604909896851} \ No newline at end of file diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug-internal.log b/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..2a54fcd290e5b92deb558bfbd29388b0300cdb6d --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug-internal.log @@ -0,0 +1,746 @@ +2025-01-21 21:56:27,645 INFO StreamThr :2658709 [internal.py:wandb_internal():85] W&B internal server running at pid: 2658709, started at: 2025-01-21 21:56:27.645113 +2025-01-21 21:56:27,646 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:56:27,656 INFO WriterThread:2658709 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/run-aq8qih16.wandb +2025-01-21 21:56:27,657 DEBUG SenderThread:2658709 [sender.py:send():379] send: header +2025-01-21 21:56:27,665 DEBUG SenderThread:2658709 [sender.py:send():379] send: run +2025-01-21 21:56:27,901 INFO SenderThread:2658709 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files +2025-01-21 21:56:27,901 INFO SenderThread:2658709 [sender.py:_start_run_threads():1188] run started: aq8qih16 with start time 1737514587.645315 +2025-01-21 21:56:27,917 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:56:27,917 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:56:27,970 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:56:27,974 DEBUG HandlerThread:2658709 [system_info.py:__init__():26] System info init +2025-01-21 21:56:27,975 DEBUG HandlerThread:2658709 [system_info.py:__init__():41] System info init done +2025-01-21 21:56:27,975 INFO HandlerThread:2658709 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:56:27,975 INFO SystemMonitor:2658709 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:56:27,975 INFO HandlerThread:2658709 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:56:27,976 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:56:27,976 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:56:27,977 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:56:27,977 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:56:27,978 INFO SystemMonitor:2658709 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:56:27,987 DEBUG HandlerThread:2658709 [system_info.py:probe():152] Probing system +2025-01-21 21:56:27,989 DEBUG HandlerThread:2658709 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:56:27,993 DEBUG HandlerThread:2658709 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:56:27,993 DEBUG HandlerThread:2658709 [system_info.py:probe():200] Probing system done +2025-01-21 21:56:27,993 DEBUG HandlerThread:2658709 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:56:27.987764', 'startedAt': '2025-01-22T02:56:27.632967', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3745.9728750000004, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.83, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.985, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 428.19886779785156}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:56:27,993 INFO HandlerThread:2658709 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:56:27,993 INFO HandlerThread:2658709 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:56:27,993 DEBUG HandlerThread:2658709 [system_info.py:_save_code():44] Saving code +2025-01-21 21:56:27,995 DEBUG HandlerThread:2658709 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:56:27,995 DEBUG HandlerThread:2658709 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:56:28,057 DEBUG HandlerThread:2658709 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:56:28,057 INFO HandlerThread:2658709 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:56:28,060 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:56:28,060 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:56:28,060 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:56:28,061 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:56:28,297 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/6lfg37oo-wandb-metadata.json +2025-01-21 21:56:28,303 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:56:28,303 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:56:28,306 DEBUG SenderThread:2658709 [sender.py:send():379] send: telemetry +2025-01-21 21:56:28,353 INFO wandb-upload_2:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/zsq0kvhv-diff.patch +2025-01-21 21:56:28,353 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:56:28,353 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:28,354 INFO HandlerThread:2658709 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/tb" + save: true +} +control { + relay_id: "b4a5101f82b94a7399ab6e78a37f9fea" +} +_info { + stream_id: "aq8qih16" +} + +2025-01-21 21:56:28,354 DEBUG HandlerThread:2658709 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:56:28,355 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:56:28,362 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/o68q7yps-code/FISH/eval_robot.py +2025-01-21 21:56:28,504 DEBUG SenderThread:2658709 [sender.py:send():379] send: tbrecord +2025-01-21 21:56:28,505 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:56:28,505 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 with policy live +2025-01-21 21:56:28,698 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/6e9difk8-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:56:28,903 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/requirements.txt +2025-01-21 21:56:28,903 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code/FISH/eval_robot.py +2025-01-21 21:56:28,903 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:28,903 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:56:28,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-metadata.json +2025-01-21 21:56:28,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/diff.patch +2025-01-21 21:56:28,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code +2025-01-21 21:56:28,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code/FISH +2025-01-21 21:56:29,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:30,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:30,902 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:31,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:32,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:33,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:33,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:34,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:35,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:36,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:36,691 DEBUG SenderThread:2658709 [sender.py:send():379] send: config +2025-01-21 21:56:37,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:38,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:38,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:38,904 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:39,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:40,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:41,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:42,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:42,906 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:43,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:56:43,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:56:43,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:43,405 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:44,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:45,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:46,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:46,907 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:47,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:48,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:49,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:49,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:50,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:51,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:52,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:52,908 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:53,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:54,309 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:54,909 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:55,245 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:56:55,309 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:56,309 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:56,910 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:56:57,309 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:58,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:56:58,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:56:58,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:56:59,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:00,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:00,453 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:00,911 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml +2025-01-21 21:57:01,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:02,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:03,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:04,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:04,912 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:05,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:05,649 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:06,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:07,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:08,310 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:09,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:10,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:11,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:11,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:12,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:12,914 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:13,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:57:13,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:57:13,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:14,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:15,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:16,312 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:16,389 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:17,311 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:18,312 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:19,312 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:20,312 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:21,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:22,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:22,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:23,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:24,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:25,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:26,314 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:27,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:27,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:27,978 DEBUG SystemMonitor:2658709 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:57:27,979 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:57:28,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:57:28,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:57:28,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:29,314 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:29,653 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:57:29,653 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 with policy now +2025-01-21 21:57:29,653 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:57:29,654 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:57:29,654 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:57:29,655 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:57:29,918 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 +2025-01-21 21:57:29,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:57:29,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos +2025-01-21 21:57:29,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 21:57:29,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media +2025-01-21 21:57:29,926 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/ne2t3wkp-media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 +2025-01-21 21:57:30,258 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/tz6rowed-media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 +2025-01-21 21:57:30,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:30,919 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:31,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:32,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:32,666 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:33,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:34,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:34,920 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:35,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:36,315 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:36,921 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:37,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:38,033 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:38,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:39,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:40,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:40,922 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:57:41,213 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/2fd02t53-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:57:41,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:42,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:42,923 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:43,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:57:43,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:57:43,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:43,373 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:44,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:45,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:46,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:46,924 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:47,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:48,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:49,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:49,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:50,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:51,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:52,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:53,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:54,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:55,225 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:57:55,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:56,317 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:56,927 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:57,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:57,979 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:57:58,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:57:58,304 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:57:58,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:57:58,928 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:57:59,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:00,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:00,385 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:01,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:02,318 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:03,319 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:04,321 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:04,930 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:05,320 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:06,178 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:06,320 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:07,320 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:08,321 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:08,932 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:09,321 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:09,792 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:58:09,792 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 with policy now +2025-01-21 21:58:09,793 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:58:09,793 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:58:09,793 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:58:09,794 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:58:09,932 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:58:09,932 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 +2025-01-21 21:58:09,933 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 21:58:10,062 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/tj1v9neo-media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 +2025-01-21 21:58:10,337 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:10,345 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/y6ppesb6-media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 +2025-01-21 21:58:10,933 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:11,325 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:11,811 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:11,933 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:12,329 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:13,307 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:58:13,307 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:58:13,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:14,328 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:14,934 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:15,327 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:16,325 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:16,935 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:17,093 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:17,325 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:18,326 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:19,326 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:19,759 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:58:19,760 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:58:19,771 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:58:19,936 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:58:19,936 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:58:20,326 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:20,936 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:21,326 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:22,327 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:22,509 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:22,937 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:23,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:24,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:24,937 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:25,331 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:26,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:27,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:27,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:27,979 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:58:28,304 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:58:28,305 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:58:28,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:29,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:30,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:31,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:32,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:33,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:33,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:34,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:35,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:36,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:37,332 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:38,330 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:38,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:39,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:40,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:41,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:42,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:43,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:58:43,305 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:58:43,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:43,402 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:44,333 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:45,338 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:45,837 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:58:45,838 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 with policy now +2025-01-21 21:58:45,842 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:58:45,842 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:58:45,843 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:58:45,843 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:58:45,945 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:58:45,946 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 +2025-01-21 21:58:45,946 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 21:58:46,095 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/nelnvo5q-media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 +2025-01-21 21:58:46,337 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:46,386 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/ei6lq6iz-media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 +2025-01-21 21:58:46,946 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:47,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:47,946 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:48,347 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:49,271 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:49,342 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:50,348 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:50,947 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:51,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:52,340 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:52,948 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:52,948 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:58:53,235 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/sxid0tpy-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:58:53,345 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:53,582 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:58:53,582 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:58:53,582 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:58:53,948 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:58:54,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:54,948 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:54,965 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:58:55,348 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:56,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:57,347 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:57,980 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:58:58,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:58:58,305 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:58:58,347 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:58:58,950 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:58:59,348 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:00,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:00,445 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:01,349 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:02,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:03,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:04,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:05,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:06,334 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:06,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:07,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:08,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:09,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:10,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:11,335 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:11,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:12,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:13,305 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:59:13,305 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:59:13,350 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:14,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:15,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:16,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:16,641 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:16,956 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:17,351 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:18,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:19,357 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:19,887 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 21:59:19,887 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 with policy now +2025-01-21 21:59:19,893 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:59:19,894 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:59:19,894 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:59:19,895 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:59:19,957 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:59:19,957 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 +2025-01-21 21:59:19,957 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 21:59:20,150 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/8ylus15t-media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 +2025-01-21 21:59:20,353 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:20,425 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/tflqghtb-media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 +2025-01-21 21:59:20,957 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:21,358 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:21,958 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:21,960 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:22,357 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:23,359 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:24,361 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:24,959 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:25,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:26,355 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:26,622 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 21:59:26,622 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:59:26,622 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:59:26,959 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:26,960 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 21:59:26,960 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 21:59:27,291 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:27,359 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:27,980 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:59:28,313 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:59:28,313 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:59:28,370 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:28,960 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:29,366 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:30,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:30,961 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:31,361 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:32,361 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:32,569 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:32,962 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 21:59:33,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:34,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:35,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:36,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:37,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:38,336 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:38,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:39,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:40,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:41,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:42,362 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:43,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:59:43,306 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:59:43,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:43,405 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:44,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:45,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:46,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:47,363 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:48,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:49,337 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:49,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:50,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:51,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:52,365 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:53,366 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:54,338 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:59:54,366 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:55,368 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:56,368 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:57,368 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:57,981 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 21:59:58,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:59:58,306 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:59:58,369 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:59,368 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:59:59,463 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:00,369 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:01,370 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:02,372 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:02,972 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:03,384 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:03,504 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 22:00:03,504 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 with policy now +2025-01-21 22:00:03,507 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:00:03,508 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:03,508 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:03,508 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:03,763 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/74h73bsl-media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 +2025-01-21 22:00:03,972 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:03,973 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 +2025-01-21 22:00:03,973 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval +2025-01-21 22:00:04,264 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/5v3wqnh9-media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 +2025-01-21 22:00:04,388 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:04,552 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:04,973 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:05,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:05,973 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:06,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:07,400 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:08,400 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:08,974 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:09,390 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:09,918 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:10,407 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:10,975 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:11,410 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:12,406 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:13,316 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:00:13,316 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:00:13,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:14,403 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:14,679 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:14,680 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:14,680 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:14,976 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:00:14,976 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:00:14,977 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:14,977 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:15,181 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:00:15,248 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/esa4nea3-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:15,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:15,468 INFO wandb-upload_0:2658709 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpyitzqrk6 +2025-01-21 22:00:15,543 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 22:00:15,555 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 22:00:15,904 INFO SenderThread:2658709 [sender.py:send_request_log_artifact():1525] logged artifact run-aq8qih16-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcwODM5MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMjM0OA==', 'latestArtifact': None}} +2025-01-21 22:00:15,904 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 22:00:15,904 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json with policy now +2025-01-21 22:00:15,905 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:15,905 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:15,905 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,905 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,906 DEBUG SenderThread:2658709 [sender.py:send():379] send: telemetry +2025-01-21 22:00:15,906 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,906 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,906 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,908 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,908 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,908 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,909 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,909 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,909 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,909 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,910 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,910 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,910 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,910 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,911 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,911 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,911 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,912 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,912 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,912 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,912 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:15,913 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:15,913 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table +2025-01-21 22:00:15,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval +2025-01-21 22:00:16,107 INFO wandb-upload_2:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/8reeflyc-media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json +2025-01-21 22:00:16,167 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/nh0voxvz-media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json +2025-01-21 22:00:16,310 INFO wandb-upload_1:2658709 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpfy9mhbrg +2025-01-21 22:00:16,377 INFO wandb-upload_2:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/f8ynsx6d-media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json +2025-01-21 22:00:16,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:16,738 INFO SenderThread:2658709 [sender.py:send_request_log_artifact():1525] logged artifact run-aq8qih16-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcwODQwNA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMjM1MQ==', 'latestArtifact': None}} +2025-01-21 22:00:16,738 DEBUG SenderThread:2658709 [sender.py:send():379] send: files +2025-01-21 22:00:16,738 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json with policy now +2025-01-21 22:00:16,738 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:16,738 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:16,738 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:16,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:16,978 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:17,009 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/mgo68mow-media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json +2025-01-21 22:00:17,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:18,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:19,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:20,393 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:21,339 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:21,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:22,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:23,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:24,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:25,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:26,339 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:26,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:27,394 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:27,981 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 22:00:28,306 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 22:00:28,306 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: stop_status +2025-01-21 22:00:28,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:29,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:30,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:31,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:31,403 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:32,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:33,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:34,395 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:35,396 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:36,396 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:36,533 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:36,577 DEBUG SenderThread:2658709 [sender.py:send():379] send: exit +2025-01-21 22:00:36,577 INFO SenderThread:2658709 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:send_exit():588] handling runtime: 248 +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:send_exit():594] send defer +2025-01-21 22:00:36,578 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,578 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 22:00:36,578 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 22:00:36,578 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 1 +2025-01-21 22:00:36,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,579 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 22:00:36,579 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:36,579 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 22:00:36,579 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 2 +2025-01-21 22:00:36,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,579 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 22:00:36,579 INFO HandlerThread:2658709 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 22:00:36,579 DEBUG SystemMonitor:2658709 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 22:00:36,579 DEBUG SystemMonitor:2658709 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 22:00:36,579 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 22:00:36,580 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 22:00:36,632 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 22:00:36,632 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 22:00:36,632 INFO HandlerThread:2658709 [interfaces.py:finish():200] Joined network monitor +2025-01-21 22:00:36,633 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:36,633 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 22:00:36,633 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 3 +2025-01-21 22:00:36,633 DEBUG SenderThread:2658709 [sender.py:send():379] send: stats +2025-01-21 22:00:36,633 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,633 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 22:00:36,633 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:36,633 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 22:00:36,634 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 4 +2025-01-21 22:00:36,634 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:36,634 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 22:00:36,986 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:38,987 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:42,686 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:42,687 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:42,687 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 22:00:42,688 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:42,688 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 22:00:42,688 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 5 +2025-01-21 22:00:42,688 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:42,688 DEBUG SenderThread:2658709 [sender.py:send():379] send: history +2025-01-21 22:00:42,688 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:42,688 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 22:00:42,688 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: summary_record +2025-01-21 22:00:42,689 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:42,690 DEBUG SenderThread:2658709 [sender.py:send():379] send: summary +2025-01-21 22:00:42,693 INFO SenderThread:2658709 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 22:00:42,694 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:42,694 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 22:00:42,694 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 6 +2025-01-21 22:00:42,694 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:42,694 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 22:00:42,694 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:42,694 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 22:00:42,697 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 22:00:42,847 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 7 +2025-01-21 22:00:42,848 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:42,848 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 22:00:42,848 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:42,848 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 22:00:42,988 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:42,989 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml +2025-01-21 22:00:43,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:45,741 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 8 +2025-01-21 22:00:45,741 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:45,741 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:45,741 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 22:00:45,742 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:45,742 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 22:00:45,742 INFO SenderThread:2658709 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 22:00:45,744 INFO SenderThread:2658709 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 22:00:45,746 INFO SenderThread:2658709 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 22:00:45,760 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 9 +2025-01-21 22:00:45,760 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:45,760 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 22:00:45,760 DEBUG SenderThread:2658709 [sender.py:send():379] send: artifact +2025-01-21 22:00:45,990 INFO Thread-12 :2658709 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:46,003 INFO SenderThread:2658709 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 22:00:46,004 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:46,004 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 22:00:46,004 INFO SenderThread:2658709 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 22:00:46,579 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml config.yaml +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json wandb-summary.json +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/requirements.txt requirements.txt +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-metadata.json wandb-metadata.json +2025-01-21 22:00:46,991 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:46,993 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log output.log +2025-01-21 22:00:46,993 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/diff.patch diff.patch +2025-01-21 22:00:46,995 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 22:00:46,996 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 media/videos/eval/2_eval_2_ee9b6581bc6fe6d9539b.mp4 +2025-01-21 22:00:46,996 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 media/videos/eval/4_eval_4_5bc7e953a75c0b8f4676.mp4 +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 media/videos/eval/1_eval_1_5e52801f26f69cbfb590.mp4 +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 media/videos/eval/0_eval_0_d41b9de2cc77d8823f41.mp4 +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 media/videos/eval/3_eval_3_9ef491d5f6f41b1a481b.mp4 +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json media/table/eval/total_success_rate_ci_6_5f51e5dd50014acd45f6.table.json +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json media/table/eval/success_rate_ci_5_50e92e293996504485a1.table.json +2025-01-21 22:00:46,997 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 10 +2025-01-21 22:00:46,997 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:46,997 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:46,998 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 22:00:46,999 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:46,999 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 22:00:46,999 INFO SenderThread:2658709 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:00:47,179 INFO wandb-upload_0:2658709 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/requirements.txt +2025-01-21 22:00:47,184 INFO wandb-upload_2:2658709 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/config.yaml +2025-01-21 22:00:47,220 INFO wandb-upload_4:2658709 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/output.log +2025-01-21 22:00:47,253 INFO wandb-upload_1:2658709 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/files/wandb-summary.json +2025-01-21 22:00:47,297 INFO wandb-upload_3:2658709 [upload_job.py:push():130] Uploaded file /tmp/tmp4c35ur6nwandb/6i9eht82-events.out.tfevents.1737514588.leonmkim-ROG-Strix-G15CS-G15CS.2658535.0 +2025-01-21 22:00:47,498 INFO Thread-11 (_thread_body):2658709 [sender.py:transition_state():614] send defer: 11 +2025-01-21 22:00:47,498 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:47,498 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 22:00:47,499 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:47,499 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 22:00:47,499 INFO SenderThread:2658709 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 22:00:47,500 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 12 +2025-01-21 22:00:47,500 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:47,500 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 22:00:47,500 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:47,500 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 22:00:47,500 INFO SenderThread:2658709 [file_stream.py:finish():601] file stream finish called +2025-01-21 22:00:47,580 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:47,713 INFO SenderThread:2658709 [file_stream.py:finish():605] file stream finish is done +2025-01-21 22:00:47,713 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 13 +2025-01-21 22:00:47,713 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:47,713 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:47,713 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 22:00:47,713 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:47,713 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 22:00:47,713 INFO SenderThread:2658709 [sender.py:transition_state():614] send defer: 14 +2025-01-21 22:00:47,713 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: defer +2025-01-21 22:00:47,713 INFO HandlerThread:2658709 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 22:00:47,714 DEBUG SenderThread:2658709 [sender.py:send():379] send: final +2025-01-21 22:00:47,714 DEBUG SenderThread:2658709 [sender.py:send():379] send: footer +2025-01-21 22:00:47,714 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: defer +2025-01-21 22:00:47,714 INFO SenderThread:2658709 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 22:00:47,714 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:47,715 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:47,715 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 22:00:47,715 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 22:00:47,715 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: server_info +2025-01-21 22:00:47,717 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 22:00:47,717 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 22:00:47,718 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 22:00:47,768 DEBUG SenderThread:2658709 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 22:00:47,768 INFO MainThread:2658709 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 22:00:47,769 INFO MainThread:2658709 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 22:00:47,769 INFO MainThread:2658709 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 22:00:47,769 DEBUG HandlerThread:2658709 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 22:00:47,769 INFO HandlerThread:2658709 [handler.py:finish():882] shutting down handler +2025-01-21 22:00:48,715 INFO WriterThread:2658709 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/run-aq8qih16.wandb +2025-01-21 22:00:48,768 INFO SenderThread:2658709 [sender.py:finish():1615] shutting down sender +2025-01-21 22:00:48,768 INFO SenderThread:2658709 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 22:00:48,769 INFO SenderThread:2658709 [file_pusher.py:join():175] waiting for file pusher diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug.log b/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..882acb80422dc50fffae91669710a24d21bae9c8 --- /dev/null +++ b/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Configure stats pid to 2658535 +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/settings +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug.log +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/wandb/run-20250121_215627-aq8qih16/logs/debug-internal.log +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_2', 'wandb_notes': '4295_2_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16'} +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:init():619] starting backend +2025-01-21 21:56:27,639 INFO MainThread:2658535 [wandb_init.py:init():623] setting up manager +2025-01-21 21:56:27,644 INFO MainThread:2658535 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:56:27,645 INFO MainThread:2658535 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:56:27,656 INFO MainThread:2658535 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:56:27,663 INFO MainThread:2658535 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:56:27,917 INFO MainThread:2658535 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:56:27,964 INFO MainThread:2658535 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:56:27,964 INFO MainThread:2658535 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:56:28,304 INFO MainThread:2658535 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:56:28,304 INFO MainThread:2658535 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:56:28,305 INFO MainThread:2658535 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:56:28,305 INFO MainThread:2658535 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:56:28,307 INFO MainThread:2658535 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:56:28,307 INFO MainThread:2658535 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/aq8qih16/tb, True +2025-01-21 21:56:36,690 INFO MainThread:2658535 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 22:00:48,770 WARNING MsgRouterThr:2658535 [router.py:message_loop():77] message_loop has been closed diff --git a/aq8qih16/wandb/run-20250121_215627-aq8qih16/run-aq8qih16.wandb b/aq8qih16/wandb/run-20250121_215627-aq8qih16/run-aq8qih16.wandb new file mode 100644 index 0000000000000000000000000000000000000000..cf2eabc67fdcaaf8a6a109a49e3db832e6313098 Binary files /dev/null and b/aq8qih16/wandb/run-20250121_215627-aq8qih16/run-aq8qih16.wandb differ diff --git a/b7vsi9bc/.hydra/config.yaml b/b7vsi9bc/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..836895e41a10194087484706499cf57e8cf16e48 --- /dev/null +++ b/b7vsi9bc/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/b7vsi9bc/.hydra/hydra.yaml b/b7vsi9bc/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d45360d058ff8b49ad7f429db1c9cb389956fd9 --- /dev/null +++ b/b7vsi9bc/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/b7vsi9bc + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/b7vsi9bc/.hydra/overrides.yaml b/b7vsi9bc/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/b7vsi9bc/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/b7vsi9bc/eval_policy.log b/b7vsi9bc/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..1572cc6da30d0a84d8cd5565489f9336405c57f5 --- /dev/null +++ b/b7vsi9bc/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 21:50:25,084][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:50:25,087][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 21:50:28,393][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a4d7fa7e5d492944ac612becf7dda4ace57b28a --- /dev/null +++ b/config.yaml @@ -0,0 +1,575 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 2 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '4295_2' +wandb_notes: 4295_2_restarted_1 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: false +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 7 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +restore_missing_checkpoint: null +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ${test.root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${test.name_of_expert_demo} + expert_dataset: ${test.expert_dataset_dirpath}/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ${test.expert_dataset_dirpath}/${test.semantic_demo_grouping_name} + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: ${test.local_snapshot_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name} + resume_wandb_run: false +feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/pkbouru1/.hydra/config.yaml b/pkbouru1/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..836895e41a10194087484706499cf57e8cf16e48 --- /dev/null +++ b/pkbouru1/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/pkbouru1/.hydra/hydra.yaml b/pkbouru1/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c789aae12e0e3a20e407c9006e70d83dda16947 --- /dev/null +++ b/pkbouru1/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/pkbouru1/.hydra/overrides.yaml b/pkbouru1/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/pkbouru1/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/pkbouru1/episode_rosbags/aligned_depth_to_color_K.npy b/pkbouru1/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/pkbouru1/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/pkbouru1/episode_rosbags/cam_tf_world.npy b/pkbouru1/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/pkbouru1/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/pkbouru1/episode_rosbags/color_K.npy b/pkbouru1/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/pkbouru1/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/pkbouru1/episode_rosbags/depth_K.npy b/pkbouru1/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/pkbouru1/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/pkbouru1/episode_rosbags/episode_2_2025-01-21-21-32-52.bag b/pkbouru1/episode_rosbags/episode_2_2025-01-21-21-32-52.bag new file mode 100644 index 0000000000000000000000000000000000000000..b6a83b67e62ed3f4bf5f484f485c04a494333d18 --- /dev/null +++ b/pkbouru1/episode_rosbags/episode_2_2025-01-21-21-32-52.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:35668d864b65f668d8031d6c783f22c1bd2ecc8a588bae6a4bb13121e4d64ec8 +size 1127153409 diff --git a/pkbouru1/eval_robot.log b/pkbouru1/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..14766d117560e34aa7725fed3cf395ab85e55539 --- /dev/null +++ b/pkbouru1/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:30:40,028][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:30:40,031][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/pkbouru1/eval_video/0_eval.mp4 b/pkbouru1/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29eef3c2ae444f612828604580f4565da060da04 --- /dev/null +++ b/pkbouru1/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:666330ccf994f224efc72df031679338093c6038ab27cd39ececbba063e52863 +size 1499560 diff --git a/pkbouru1/eval_video/1_eval.mp4 b/pkbouru1/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85dfbc64200051a8d27ee543e5d760908dd83b49 --- /dev/null +++ b/pkbouru1/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:335a125e2c942caab269e60c6f84cc361dd70bdd7bfca6e1f54c685e48dbc945 +size 1510389 diff --git a/pkbouru1/eval_video/2_eval.mp4 b/pkbouru1/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49e645538627d6a269c6eec3b00865c9b81f8616 --- /dev/null +++ b/pkbouru1/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e61939f3ab9157666d678bbea9367430791d81d209d743912d91ca8ad503be8 +size 768084 diff --git a/pkbouru1/eval_video/3_eval.mp4 b/pkbouru1/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4c5bdbfb31f6f8aa4eece6e6d499811e7f909dd2 --- /dev/null +++ b/pkbouru1/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf7dc702773eb47d1f7eeaa3329055a85ba84c45cee39fa992f47fcc743ae867 +size 1503564 diff --git a/pkbouru1/eval_video/4_eval.mp4 b/pkbouru1/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d4202053ebb18b6bcff07392bb9249c1d90ead40 --- /dev/null +++ b/pkbouru1/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fab844ed0fc1ded78b7730be443ba91c87c2f2d31187b48f18f1675a7f0d9c7 +size 1507124 diff --git a/pkbouru1/tb/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 b/pkbouru1/tb/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 new file mode 100644 index 0000000000000000000000000000000000000000..7e625ca11b6be40308984560c1e903ef8a6ce2a5 --- /dev/null +++ b/pkbouru1/tb/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:653d5c6b0d9e0dc18e6f83324b5dd0a0cccc20ae4cf301efaae0a8bd73045c25 +size 6539 diff --git a/pkbouru1/wandb/debug-internal.log b/pkbouru1/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..41a67322baa764df485c9bc6cab0a4fd1213f985 --- /dev/null +++ b/pkbouru1/wandb/debug-internal.log @@ -0,0 +1,791 @@ +2025-01-21 21:30:45,832 INFO StreamThr :2651916 [internal.py:wandb_internal():85] W&B internal server running at pid: 2651916, started at: 2025-01-21 21:30:45.832388 +2025-01-21 21:30:45,834 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:30:45,844 INFO WriterThread:2651916 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/run-pkbouru1.wandb +2025-01-21 21:30:45,844 DEBUG SenderThread:2651916 [sender.py:send():379] send: header +2025-01-21 21:30:45,852 DEBUG SenderThread:2651916 [sender.py:send():379] send: run +2025-01-21 21:30:46,167 INFO SenderThread:2651916 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files +2025-01-21 21:30:46,167 INFO SenderThread:2651916 [sender.py:_start_run_threads():1188] run started: pkbouru1 with start time 1737513045.832786 +2025-01-21 21:30:46,186 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:30:46,186 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:30:46,262 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:30:46,267 DEBUG HandlerThread:2651916 [system_info.py:__init__():26] System info init +2025-01-21 21:30:46,267 DEBUG HandlerThread:2651916 [system_info.py:__init__():41] System info init done +2025-01-21 21:30:46,267 INFO HandlerThread:2651916 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:30:46,267 INFO SystemMonitor:2651916 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:30:46,268 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:30:46,268 INFO HandlerThread:2651916 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:30:46,269 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:30:46,269 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:30:46,270 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:30:46,270 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:30:46,281 DEBUG HandlerThread:2651916 [system_info.py:probe():152] Probing system +2025-01-21 21:30:46,283 DEBUG HandlerThread:2651916 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:30:46,288 DEBUG HandlerThread:2651916 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:30:46,288 DEBUG HandlerThread:2651916 [system_info.py:probe():200] Probing system done +2025-01-21 21:30:46,288 DEBUG HandlerThread:2651916 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:30:46.281548', 'startedAt': '2025-01-22T02:30:45.820874', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.7545, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 381.4694061279297}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:30:46,288 INFO HandlerThread:2651916 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:30:46,288 INFO HandlerThread:2651916 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:30:46,288 DEBUG HandlerThread:2651916 [system_info.py:_save_code():44] Saving code +2025-01-21 21:30:46,289 DEBUG HandlerThread:2651916 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:30:46,289 DEBUG HandlerThread:2651916 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:30:46,352 DEBUG HandlerThread:2651916 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:30:46,353 INFO HandlerThread:2651916 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:30:46,360 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:30:46,360 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:30:46,361 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:30:46,361 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:30:46,589 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:30:46,589 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:30:46,590 DEBUG SenderThread:2651916 [sender.py:send():379] send: telemetry +2025-01-21 21:30:46,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:46,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:30:46,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:30:46,633 INFO HandlerThread:2651916 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/tb" + save: true +} +control { + relay_id: "f6d28eb1c12a46838862bea7e051a09a" +} +_info { + stream_id: "pkbouru1" +} + +2025-01-21 21:30:46,633 DEBUG HandlerThread:2651916 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:30:46,652 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ea9nclmn-diff.patch +2025-01-21 21:30:46,658 INFO wandb-upload_1:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/f2w68x3r-code/FISH/eval_robot.py +2025-01-21 21:30:46,659 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/tpdhf4pz-wandb-metadata.json +2025-01-21 21:30:46,718 DEBUG SenderThread:2651916 [sender.py:send():379] send: tbrecord +2025-01-21 21:30:46,718 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:30:46,718 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 with policy live +2025-01-21 21:30:46,897 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/dfpwy5ze-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:30:47,169 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-metadata.json +2025-01-21 21:30:47,169 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/requirements.txt +2025-01-21 21:30:47,169 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:30:47,169 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code/FISH/eval_robot.py +2025-01-21 21:30:47,170 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/diff.patch +2025-01-21 21:30:47,170 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:30:47,170 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code/FISH +2025-01-21 21:30:47,170 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code +2025-01-21 21:30:47,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:48,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:49,168 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:30:49,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:50,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:51,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:51,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:52,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:53,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:54,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:55,016 DEBUG SenderThread:2651916 [sender.py:send():379] send: config +2025-01-21 21:30:55,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:56,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:56,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:57,172 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:30:57,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:58,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:59,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:00,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:01,173 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:01,589 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:31:01,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:31:01,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:01,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:02,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:03,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:04,593 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:05,174 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:05,593 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:06,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:07,175 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:07,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:07,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:08,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:09,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:10,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:11,176 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:11,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:12,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:12,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:13,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:14,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:15,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:16,589 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:31:16,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:31:16,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:17,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:17,719 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:18,177 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml +2025-01-21 21:31:18,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:19,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:20,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:21,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:22,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:23,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:23,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:24,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:25,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:26,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:27,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:28,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:28,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:29,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:30,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:31,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:31:31,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:31:31,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:32,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:33,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:33,677 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:34,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:35,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:36,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:37,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:38,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:39,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:39,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:40,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:41,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:42,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:43,184 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:43,609 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:44,573 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:31:44,574 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:31:44,574 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 with policy now +2025-01-21 21:31:44,575 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:31:44,575 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:31:44,575 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:31:44,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:44,833 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/83plspd5-media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:31:45,469 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/p2eq6pp7-media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 +2025-01-21 21:31:45,584 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:45,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:46,270 DEBUG SystemMonitor:2651916 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:31:46,271 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:31:46,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:31:46,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:31:46,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:47,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:47,599 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:48,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:49,186 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:49,600 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:50,600 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:50,953 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:51,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:52,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:53,187 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:53,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:54,188 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:31:54,489 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/oadx6w7s-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:31:54,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:55,188 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:55,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:56,025 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:56,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:57,189 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:57,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:58,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:59,190 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:59,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:00,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:01,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:32:01,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:32:01,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:01,705 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:02,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:03,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:04,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:05,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:06,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:07,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:07,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:08,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:09,604 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:10,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:11,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:12,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:12,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:13,604 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:14,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:15,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:16,271 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:32:16,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:32:16,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:32:16,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:17,195 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:17,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:17,662 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:18,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:19,196 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:19,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:20,606 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:21,606 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:22,606 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:23,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:23,606 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:24,608 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:25,607 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:26,607 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:27,607 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:28,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:28,607 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:29,608 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:30,608 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:31,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:32:31,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:32:31,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:32,608 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:33,611 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:33,714 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:33,757 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:32:33,757 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:32:33,758 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 with policy now +2025-01-21 21:32:33,758 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:32:33,758 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:32:33,759 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:32:33,981 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ig7c3sos-media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 +2025-01-21 21:32:34,201 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:32:34,201 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 +2025-01-21 21:32:34,201 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:32:34,484 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/mj0ozn2x-media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 +2025-01-21 21:32:34,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:35,201 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:35,615 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:36,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:37,618 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:38,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:39,203 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:39,417 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:39,621 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:40,615 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:41,203 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:41,615 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:42,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:43,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:44,593 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:44,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:45,618 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:46,272 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:32:46,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:32:46,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:32:46,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:47,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:48,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:49,063 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:32:49,063 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:32:49,063 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:32:49,206 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:32:49,206 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:32:49,618 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:50,537 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:50,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:51,206 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:51,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:52,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:53,207 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:53,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:54,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:55,208 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:55,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:55,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:56,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:57,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:58,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:59,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:00,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:01,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:33:01,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:33:01,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:01,710 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:02,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:03,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:04,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:05,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:06,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:07,593 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:07,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:08,628 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:08,984 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:33:08,984 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 with policy now +2025-01-21 21:33:08,991 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:33:08,991 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:33:08,992 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:33:08,992 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:33:09,198 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/zvcmr5ax-media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 +2025-01-21 21:33:09,212 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:09,213 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:33:09,213 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 +2025-01-21 21:33:09,213 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:33:09,477 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ty0mj2be-media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 +2025-01-21 21:33:09,631 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:10,625 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:11,213 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:11,627 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:12,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:12,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:13,627 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:14,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:15,214 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:15,625 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:16,272 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:33:16,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:33:16,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:33:16,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:17,215 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:17,627 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:17,720 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:18,628 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:19,628 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:20,628 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:21,627 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:22,632 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:23,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:23,636 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:24,636 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:25,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:26,637 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:27,219 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:33:27,469 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ol83ayog-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:33:27,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:27,887 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:33:27,887 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:33:27,887 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:33:28,219 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:33:28,646 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:28,993 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:29,219 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:29,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:30,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:31,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:33:31,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:33:31,639 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:32,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:33,221 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:33,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:34,639 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:34,732 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:35,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:36,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:37,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:38,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:39,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:40,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:40,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:41,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:42,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:43,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:44,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:45,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:45,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:46,273 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:33:46,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:33:46,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:33:46,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:47,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:48,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:49,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:50,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:50,742 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:51,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:52,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:53,227 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:53,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:54,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:55,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:56,437 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:56,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:57,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:58,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:59,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:00,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:01,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:34:01,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:34:01,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:01,744 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:02,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:03,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:04,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:05,653 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:06,650 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:06,761 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:34:06,761 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 with policy now +2025-01-21 21:34:06,761 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:34:06,763 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:06,764 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:34:06,765 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:34:06,770 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:34:06,990 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/x2vu43wd-media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 +2025-01-21 21:34:07,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:07,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:34:07,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 +2025-01-21 21:34:07,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:34:07,504 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/im2a57kj-media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 +2025-01-21 21:34:07,650 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:08,649 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:09,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:09,660 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:10,651 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:11,233 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:11,652 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:12,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:12,653 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:13,234 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:13,652 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:14,657 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:15,235 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:15,657 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:16,273 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:34:16,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:34:16,604 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:34:16,664 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:17,659 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:17,704 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:17,947 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:34:17,947 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:34:17,948 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:34:18,236 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:34:18,236 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:34:18,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:19,236 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:19,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:20,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:21,237 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:21,673 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:22,667 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:22,893 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:23,237 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:23,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:24,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:25,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:26,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:27,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:28,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:28,865 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:29,240 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:29,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:30,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:31,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:34:31,597 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:34:31,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:32,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:33,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:34,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:34,748 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:35,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:36,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:37,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:38,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:39,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:40,613 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:40,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:41,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:42,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:43,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:44,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:45,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:46,274 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:34:46,274 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:46,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:34:46,597 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:34:46,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:47,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:48,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:49,246 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:49,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:50,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:51,612 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:51,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:52,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:53,249 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:53,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:54,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:55,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:56,613 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:56,684 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:57,250 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:57,579 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:34:57,579 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 with policy now +2025-01-21 21:34:57,584 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:34:57,659 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:34:57,659 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:34:57,673 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:57,686 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:34:57,808 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/yj9tkrjw-media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 +2025-01-21 21:34:58,251 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:34:58,251 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 +2025-01-21 21:34:58,251 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:34:58,544 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/drleyppi-media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 +2025-01-21 21:34:58,675 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:59,251 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:59,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:00,683 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:01,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:35:01,603 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:35:01,666 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:01,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:02,677 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:03,253 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:03,684 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:04,679 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:05,254 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:05,682 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:06,678 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:07,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:07,684 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:08,686 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:09,693 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:10,021 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:35:10,021 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:10,022 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:10,238 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:35:10,238 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table +2025-01-21 21:35:10,398 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:35:10,442 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/6i5c7b10-media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json +2025-01-21 21:35:10,546 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/o6w4rf8n-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:10,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:10,748 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:35:10,782 INFO wandb-upload_1:2651916 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpaetffev_ +2025-01-21 21:35:10,793 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:35:11,207 INFO SenderThread:2651916 [sender.py:send_request_log_artifact():1525] logged artifact run-pkbouru1-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTA1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NzUyOQ==', 'latestArtifact': None}} +2025-01-21 21:35:11,207 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:35:11,207 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json with policy now +2025-01-21 21:35:11,207 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:35:11,208 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,208 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,208 DEBUG SenderThread:2651916 [sender.py:send():379] send: telemetry +2025-01-21 21:35:11,208 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,212 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,212 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,213 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,213 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,213 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,213 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,214 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,214 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,214 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,214 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,215 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,215 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,215 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,215 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,216 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,216 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,216 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,216 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,217 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,217 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,217 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,217 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:35:11,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:11,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:11,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:11,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json +2025-01-21 21:35:11,257 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval +2025-01-21 21:35:11,460 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/0rml6o24-media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json +2025-01-21 21:35:11,483 INFO wandb-upload_1:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ce38462j-media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json +2025-01-21 21:35:11,604 INFO wandb-upload_2:2651916 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpuuksyuk_ +2025-01-21 21:35:11,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:11,999 INFO SenderThread:2651916 [sender.py:send_request_log_artifact():1525] logged artifact run-pkbouru1-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTA2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NzUzMA==', 'latestArtifact': None}} +2025-01-21 21:35:11,999 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:35:11,999 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json with policy now +2025-01-21 21:35:11,999 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:35:12,000 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:12,000 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:12,249 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/hmyxkqwf-media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json +2025-01-21 21:35:12,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:12,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:13,001 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:13,257 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:13,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:14,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:15,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:16,274 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:35:16,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:35:16,597 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:35:16,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:17,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:18,665 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:18,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:19,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:20,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:21,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:22,682 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:23,682 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:24,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:24,682 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:25,409 DEBUG SenderThread:2651916 [sender.py:send():379] send: exit +2025-01-21 21:35:25,409 INFO SenderThread:2651916 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:35:25,410 INFO SenderThread:2651916 [sender.py:send_exit():588] handling runtime: 279 +2025-01-21 21:35:25,411 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:25,411 INFO SenderThread:2651916 [sender.py:send_exit():594] send defer +2025-01-21 21:35:25,411 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,411 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:35:25,411 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:25,411 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:35:25,411 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:35:25,411 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,411 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:35:25,412 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:25,412 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:35:25,412 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:35:25,412 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,412 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:35:25,412 INFO HandlerThread:2651916 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:35:25,412 DEBUG SystemMonitor:2651916 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:35:25,412 DEBUG SystemMonitor:2651916 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:35:25,413 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:35:25,413 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:35:25,485 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:35:25,485 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:35:25,486 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:35:25,486 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:25,486 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:35:25,486 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:35:25,486 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:35:25,486 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,486 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:35:25,487 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:25,487 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:35:25,487 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:35:25,487 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,487 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:35:26,262 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:27,263 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:32,029 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:32,029 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:32,029 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:35:32,030 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:32,030 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:35:32,030 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:35:32,031 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:32,031 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:35:32,031 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:32,031 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:32,031 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:35:32,032 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:32,032 DEBUG SenderThread:2651916 [sender.py:send():379] send: summary +2025-01-21 21:35:32,036 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:32,036 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:32,036 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:35:32,036 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:35:32,036 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:32,036 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:35:32,037 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:32,037 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:35:32,040 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:32,149 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:35:32,149 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:32,149 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:35:32,150 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:32,150 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:35:32,274 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml +2025-01-21 21:35:32,274 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:32,411 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:33,056 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:35:33,056 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:33,056 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:33,056 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:35:33,057 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:33,057 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:35:33,057 INFO SenderThread:2651916 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:35:33,059 INFO SenderThread:2651916 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:35:33,061 INFO SenderThread:2651916 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:35:33,076 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:35:33,077 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:33,077 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:35:33,077 DEBUG SenderThread:2651916 [sender.py:send():379] send: artifact +2025-01-21 21:35:33,274 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:33,411 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:33,535 INFO wandb-upload_0:2651916 [upload_job.py:push():85] Skipped uploading /tmp/tmpjb9r83j6/wandb-job.json +2025-01-21 21:35:33,535 INFO wandb-upload_1:2651916 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmphi4_s9v7 +2025-01-21 21:35:33,630 INFO wandb-upload_2:2651916 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppc454fha +2025-01-21 21:35:34,442 INFO SenderThread:2651916 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3MzQyMw==', 'versionIndex': 455}}} +2025-01-21 21:35:34,442 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:34,442 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:35:34,442 INFO SenderThread:2651916 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:35:35,275 INFO SenderThread:2651916 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files +2025-01-21 21:35:35,275 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml config.yaml +2025-01-21 21:35:35,275 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json wandb-summary.json +2025-01-21 21:35:35,275 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/requirements.txt requirements.txt +2025-01-21 21:35:35,279 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:35:35,279 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:35,279 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log output.log +2025-01-21 21:35:35,281 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/diff.patch diff.patch +2025-01-21 21:35:35,281 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:35:35,283 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 +2025-01-21 21:35:35,283 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 +2025-01-21 21:35:35,283 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 +2025-01-21 21:35:35,284 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 +2025-01-21 21:35:35,284 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 +2025-01-21 21:35:35,284 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json +2025-01-21 21:35:35,285 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json +2025-01-21 21:35:35,285 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:35:35,285 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:35,285 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,285 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:35:35,285 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,285 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:35:35,285 INFO SenderThread:2651916 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:35:35,412 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:35,412 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:35,457 INFO wandb-upload_1:2651916 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:35,480 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml +2025-01-21 21:35:35,502 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/requirements.txt +2025-01-21 21:35:35,505 INFO wandb-upload_4:2651916 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:35,601 INFO wandb-upload_3:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/e2j3ddfv-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:35,801 INFO Thread-11 (_thread_body):2651916 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:35:35,801 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,801 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:35:35,802 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,802 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:35:35,802 INFO SenderThread:2651916 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:35:35,803 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:35:35,803 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,803 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:35:35,803 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,803 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:35:35,803 INFO SenderThread:2651916 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:35:35,872 INFO SenderThread:2651916 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:35:35,872 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:35:35,872 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,872 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:35:35,873 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,873 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:35:35,873 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:35:35,873 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,873 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:35:35,873 DEBUG SenderThread:2651916 [sender.py:send():379] send: final +2025-01-21 21:35:35,873 DEBUG SenderThread:2651916 [sender.py:send():379] send: footer +2025-01-21 21:35:35,873 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,873 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:35:35,874 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:35,874 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:35,875 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:35:35,875 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:35,875 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:35:35,875 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:35:35,877 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:35:35,877 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:35,925 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:35,925 INFO MainThread:2651916 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:35:35,925 INFO MainThread:2651916 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:35:35,926 INFO MainThread:2651916 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:35:35,926 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:35:35,926 INFO HandlerThread:2651916 [handler.py:finish():882] shutting down handler +2025-01-21 21:35:36,875 INFO WriterThread:2651916 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/run-pkbouru1.wandb +2025-01-21 21:35:36,925 INFO SenderThread:2651916 [sender.py:finish():1615] shutting down sender +2025-01-21 21:35:36,925 INFO SenderThread:2651916 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:35:36,925 INFO SenderThread:2651916 [file_pusher.py:join():175] waiting for file pusher diff --git a/pkbouru1/wandb/debug.log b/pkbouru1/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..43f7eec06327a97ef6bbb9b0ee8b87769c98e767 --- /dev/null +++ b/pkbouru1/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Configure stats pid to 2651732 +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/settings +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug.log +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug-internal.log +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_2', 'wandb_notes': '4295_2_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1'} +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:init():619] starting backend +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:init():623] setting up manager +2025-01-21 21:30:45,831 INFO MainThread:2651732 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:30:45,832 INFO MainThread:2651732 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:30:45,843 INFO MainThread:2651732 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:30:45,849 INFO MainThread:2651732 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:30:46,185 INFO MainThread:2651732 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:30:46,257 INFO MainThread:2651732 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:30:46,258 INFO MainThread:2651732 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:30:46,589 INFO MainThread:2651732 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:30:46,589 INFO MainThread:2651732 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:30:46,589 INFO MainThread:2651732 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:30:46,589 INFO MainThread:2651732 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:30:46,592 INFO MainThread:2651732 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:30:46,592 INFO MainThread:2651732 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/tb, True +2025-01-21 21:30:55,015 INFO MainThread:2651732 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:35:36,927 WARNING MsgRouterThr:2651732 [router.py:message_loop():77] message_loop has been closed diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code/FISH/eval_robot.py b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e23d45bcde6eb5bc7712cd20a510416735e546e --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 2 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_2' +wandb_notes: + desc: null + value: 4295_2_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 2 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737513045 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/diff.patch b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..d3a5568bb3f03cd705321e234cb5e34a755166b7 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/diff.patch @@ -0,0 +1,25 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c37123f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -144,9 +144,9 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' ++# wandb_run_id: '4295_0' + # wandb_run_id: '4295_1' # seed 1 +-# wandb_run_id: '4295_2' # seed 2 ++wandb_run_id: '4295_2' # seed 2 + + # RGBD+mask+act history + # wandb_run_id: '1009_1' # seed 1 diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json new file mode 100644 index 0000000000000000000000000000000000000000..7385bb646db4aef7fea3e6d2a5ca0857a1014820 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.28238606457294174, 0.9961289389929335, 0.335221469435269, 0.9917948543041831, 0.4709016382752441, 0.9765643103957722, 0.43461479247924073, 0.9410526714336915]]} \ No newline at end of file diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9e36d2c20499cefb9920e6180db3923d6e75567d --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[4, 5, 0.8, 0.37826180460503606, 0.9959998055300534, 0.42814075948245045, 0.9923493171988695, 0.41530466081405826, 0.982368674032774, 0.4757394716192128, 0.9305819905433181]]} \ No newline at end of file diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..29eef3c2ae444f612828604580f4565da060da04 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:666330ccf994f224efc72df031679338093c6038ab27cd39ececbba063e52863 +size 1499560 diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..85dfbc64200051a8d27ee543e5d760908dd83b49 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:335a125e2c942caab269e60c6f84cc361dd70bdd7bfca6e1f54c685e48dbc945 +size 1510389 diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..49e645538627d6a269c6eec3b00865c9b81f8616 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e61939f3ab9157666d678bbea9367430791d81d209d743912d91ca8ad503be8 +size 768084 diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..4c5bdbfb31f6f8aa4eece6e6d499811e7f909dd2 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf7dc702773eb47d1f7eeaa3329055a85ba84c45cee39fa992f47fcc743ae867 +size 1503564 diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d4202053ebb18b6bcff07392bb9249c1d90ead40 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fab844ed0fc1ded78b7730be443ba91c87c2f2d31187b48f18f1675a7f0d9c7 +size 1507124 diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..32b0a6296adb618dccb1a1ddffe8df013ed508aa --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log @@ -0,0 +1,271 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737513055.016721]: resetting environment +[INFO] [1737513055.020782]: cleared current plan +[INFO] [1737513055.021930]: moving to home +[INFO] [1737513056.024227]: reached home +[INFO] [1737513056.025937]: reset action history +[INFO] [1737513058.737784]: environment reset +Starting episode 0 +[INFO] [1737513058.743603]: resetting environment +[INFO] [1737513058.747783]: cleared current plan +[INFO] [1737513058.748520]: moving to home +[INFO] [1737513059.751565]: reached home +[INFO] [1737513059.752916]: reset action history +[INFO] [1737513062.464556]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737513064.920185]: resetting environment +[INFO] [1737513064.924665]: cleared current plan +[INFO] [1737513064.926697]: moving to home +[INFO] [1737513065.929661]: reached home +[INFO] [1737513065.932383]: reset action history +[INFO] [1737513068.696408]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/episode_rosbags/episode_0_2025-01-21-21-31-08.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2651732.log +started roslaunch server http://158.130.50.37:46797/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2652026] +process[print_text-2]: started with pid [2652050] +process[rosbag_record-3]: started with pid [2652051] +started bagging! +For topic gripper_width: timestamp difference is 105889905 for nearest: 1737513069579109538 - target: 1737513069473219633 at idx 11 +For topic gripper_width: timestamp difference is -105462075 for nearest: 1737513100512488841 - target: 1737513100617950916 at idx 249 +[INFO] [1737513103.564363]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1737513104.359342]: Stored episode 1. +[INFO] [1737513104.361286]: Saving video... +[INFO] [1737513104.557936]: Video saved! +Starting episode 1 +[INFO] [1737513104.574817]: resetting environment +[INFO] [1737513104.583193]: cleared current plan +[INFO] [1737513104.583569]: moving to home +[INFO] [1737513108.187014]: reached home +[INFO] [1737513108.187641]: reset action history +[INFO] [1737513110.952914]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513113.979339]: resetting environment +[INFO] [1737513113.982499]: cleared current plan +[INFO] [1737513113.982718]: moving to home +[INFO] [1737513114.984219]: reached home +[INFO] [1737513114.985117]: reset action history +[INFO] [1737513117.695398]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/episode_rosbags/episode_1_2025-01-21-21-31-57.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2651732.log +started roslaunch server http://158.130.50.37:37125/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2652172] +process[print_text-5]: started with pid [2652173] +process[rosbag_record-6]: started with pid [2652174] +started bagging! +For topic gripper_width: timestamp difference is -120589972 for nearest: 1737513135112489700 - target: 1737513135233079672 at idx 207 +For topic gripper_width: timestamp difference is -117879254 for nearest: 1737513136412485259 - target: 1737513136530364513 at idx 221 +[INFO] [1737513152.399657]: Storing episode... +[rosbag_record-6] killing on exit[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1737513153.142701]: Stored episode 2. +[INFO] [1737513153.143050]: Saving video... +[INFO] [1737513153.711161]: Video saved! +Starting episode 2 +[INFO] [1737513153.749953]: resetting environment +[INFO] [1737513153.764082]: cleared current plan +[INFO] [1737513153.765060]: moving to home +[INFO] [1737513157.368480]: reached home +[INFO] [1737513157.373430]: reset action history +[INFO] [1737513160.158218]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513168.479123]: resetting environment +[INFO] [1737513168.492903]: cleared current plan +[INFO] [1737513168.493352]: moving to home +[INFO] [1737513169.494949]: reached home +[INFO] [1737513169.495380]: reset action history +[INFO] [1737513172.204135]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/episode_rosbags/episode_2_2025-01-21-21-32-52.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2651732.log +started roslaunch server http://158.130.50.37:45839/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2652316] +process[print_text-8]: started with pid [2652317] +process[rosbag_record-9]: started with pid [2652318] +started bagging! +preempting episode +[INFO] [1737513187.732378]: Storing episode... +[print_text-8] killing on exit +[pub_text-7] killing on exit +[rosbag_record-9] killing on exit +[INFO] [1737513188.458901]: Stored episode 3. +[INFO] [1737513188.464692]: Saving video... +[INFO] [1737513188.953926]: Video saved! +Starting episode 3 +[INFO] [1737513188.995439]: resetting environment +[INFO] [1737513189.008353]: cleared current plan +[INFO] [1737513189.014503]: moving to home +[WARN] [1737513188.912479]: Plan exhausted +[WARN] [1737513188.951789]: Plan exhausted +[WARN] [1737513188.992735]: Plan exhausted +[INFO] [1737513192.624935]: reached home +[INFO] [1737513192.629981]: reset action history +[INFO] [1737513195.424649]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513206.915635]: resetting environment +[INFO] [1737513206.941174]: cleared current plan +[INFO] [1737513206.941800]: moving to home +[INFO] [1737513207.944738]: reached home +[INFO] [1737513207.946351]: reset action history +[INFO] [1737513210.725608]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/episode_rosbags/episode_3_2025-01-21-21-33-30.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2651732.log +started roslaunch server http://158.130.50.37:35067/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2652483] +process[print_text-11]: started with pid [2652484] +process[rosbag_record-12]: started with pid [2652485] +started bagging! +For topic gripper_width: timestamp difference is 102820293 for nearest: 1737513232379152671 - target: 1737513232276332378 at idx 249 +[INFO] [1737513245.608925]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1737513246.456279]: Stored episode 4. +[INFO] [1737513246.458267]: Saving video... +[INFO] [1737513246.717299]: Video saved! +Starting episode 4 +[INFO] [1737513246.773869]: resetting environment +[INFO] [1737513246.818985]: cleared current plan +[INFO] [1737513246.822672]: moving to home +[INFO] [1737513250.437121]: reached home +[INFO] [1737513250.441936]: reset action history +[INFO] [1737513253.233171]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737513257.742969]: resetting environment +[INFO] [1737513257.775158]: cleared current plan +[INFO] [1737513257.778177]: moving to home +[INFO] [1737513258.785630]: reached home +[INFO] [1737513258.788907]: reset action history +[INFO] [1737513261.572784]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/episode_rosbags/episode_4_2025-01-21-21-34-21.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2651732.log +started roslaunch server http://158.130.50.37:38569/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2652628] +process[print_text-14]: started with pid [2652629] +process[rosbag_record-15]: started with pid [2652630] +started bagging! +For topic gripper_width: timestamp difference is -201785993 for nearest: 1737513267512487221 - target: 1737513267714273214 at idx 88 +For topic gripper_width: timestamp difference is 114042984 for nearest: 1737513287579114424 - target: 1737513287465071440 at idx 249 +For topic gripper_width: timestamp difference is 116586386 for nearest: 1737513290679153143 - target: 1737513290562566757 at idx 249 +For topic gripper_width: timestamp difference is 121098227 for nearest: 1737513296179167695 - target: 1737513296058069468 at idx 249 +[INFO] [1737513296.426429]: Storing episode... +[pub_text-13] killing on exit[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[INFO] [1737513297.214400]: Stored episode 5. +[INFO] [1737513297.216075]: Saving video... +[INFO] [1737513297.516892]: Video saved! +[INFO] [1737513297.585411]: resetting environment +[INFO] [1737513297.637296]: cleared current plan +[INFO] [1737513297.637585]: moving to home +[INFO] [1737513301.242436]: reached home +[INFO] [1737513301.251354]: reset action history +[INFO] [1737513304.047412]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/requirements.txt b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-metadata.json b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..1d391b36c1ec7fb603ae397b0c2fd2fb25bc09aa --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:30:46.281548", + "startedAt": "2025-01-22T02:30:45.820874", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.7545, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4500.003, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.003, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 381.4694061279297 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b74a7616f694f58a23aea1034418d12834744508 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "666330ccf994f224efc72df031679338093c6038ab27cd39ececbba063e52863", "size": 1499560, "path": "media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4"}, "global_step": 590, "_timestamp": 1737513310.6648507, "_runtime": 264.8320646286011, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "335a125e2c942caab269e60c6f84cc361dd70bdd7bfca6e1f54c685e48dbc945", "size": 1510389, "path": "media/videos/eval/1_eval_1_335a125e2c942caab269.mp4"}, "eval/num_success": 4.0, "episode": 5.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.28238606452941895, "eval/success_rate_ub_0.01": 0.9961289167404175, "eval/success_rate_lb_0.025": 0.33522146940231323, "eval/success_rate_ub_0.025": 0.9917948246002197, "eval/success_rate_lb_0.05": 0.47090163826942444, "eval/success_rate_ub_0.05": 0.9765642881393433, "eval/success_rate_lb_0.1": 0.43461480736732483, "eval/success_rate_ub_0.1": 0.9410526752471924, "eval/2_eval": {"_type": "video-file", "sha256": "6e61939f3ab9157666d678bbea9367430791d81d209d743912d91ca8ad503be8", "size": 768084, "path": "media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "cf7dc702773eb47d1f7eeaa3329055a85ba84c45cee39fa992f47fcc743ae867", "size": 1503564, "path": "media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "4fab844ed0fc1ded78b7730be443ba91c87c2f2d31187b48f18f1675a7f0d9c7", "size": 1507124, "path": "media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "6f7b04d7609ef203f175343e65303f3512b1be412d1520f16fe229e93c2d47e4", "size": 399, "artifact_path": "wandb-client-artifact://gfbytesioqvsacx08112j1oucok0smc4m7gb3gzt157tsf1d5no607o1pt6zf4l8znqyr3ijju5dh5r1qz30h6h2r6vva490lkwq8tuu059jxfhnaxplxlrw3cb2wump/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://vigvrav3g9809sf7upaqwfpfepclatvtnbz869jdqd4m8jqvxzyorbye6hxpbjcv5bzg1cty7c27b8l2q1dp9x66x7g46nlbvkmitiglxinowlqgpxh5fepfu5xo62rd:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 4, "total_num_episodes": 5, "total_success_rate": 0.8, "total_success_rate_lb_0.01": 0.37826180460503606, "total_success_rate_ub_0.01": 0.9959998055300534, "total_success_rate_lb_0.025": 0.42814075948245045, "total_success_rate_ub_0.025": 0.9923493171988695, "total_success_rate_lb_0.05": 0.41530466081405826, "total_success_rate_ub_0.05": 0.982368674032774, "total_success_rate_lb_0.1": 0.4757394716192128, "total_success_rate_ub_0.1": 0.9305819905433181, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "24085c194cbcf9ae6df8765851ec046b54e68b53bb4231d8a1b59793abab6d83", "size": 505, "artifact_path": "wandb-client-artifact://1lz68sjiugdywcbja901kcfvfjpwhhwj0j0izyip2dzbzxvxntnbzntyyjlegb4wzhiw2h77jk23uhjyde505kq82kggk3uhnmhv0r1zran879g4dzo0niq996lj58nf/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://mujxz6r48dngp4jxoz7f7l0islkd3rygctc6hd958kiky7mitzadprfssz0lrzgcse86bsgj5j9epe7bt6okiywc23pnsfxvbu4zionf5h5wyknpw1prqajl1gzq7jn5:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 279}, "eval/total_success_rate": 0.800000011920929, "eval/total_success_rate_lb_0.01": 0.3782618045806885, "eval/total_success_rate_ub_0.01": 0.995999813079834, "eval/total_success_rate_lb_0.025": 0.4281407594680786, "eval/total_success_rate_ub_0.025": 0.9923493266105652, "eval/total_success_rate_lb_0.05": 0.41530466079711914, "eval/total_success_rate_ub_0.05": 0.9823686480522156, "eval/total_success_rate_lb_0.1": 0.4757394790649414, "eval/total_success_rate_ub_0.1": 0.9305819869041443} \ No newline at end of file diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug-internal.log b/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..41a67322baa764df485c9bc6cab0a4fd1213f985 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug-internal.log @@ -0,0 +1,791 @@ +2025-01-21 21:30:45,832 INFO StreamThr :2651916 [internal.py:wandb_internal():85] W&B internal server running at pid: 2651916, started at: 2025-01-21 21:30:45.832388 +2025-01-21 21:30:45,834 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:30:45,844 INFO WriterThread:2651916 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/run-pkbouru1.wandb +2025-01-21 21:30:45,844 DEBUG SenderThread:2651916 [sender.py:send():379] send: header +2025-01-21 21:30:45,852 DEBUG SenderThread:2651916 [sender.py:send():379] send: run +2025-01-21 21:30:46,167 INFO SenderThread:2651916 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files +2025-01-21 21:30:46,167 INFO SenderThread:2651916 [sender.py:_start_run_threads():1188] run started: pkbouru1 with start time 1737513045.832786 +2025-01-21 21:30:46,186 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:30:46,186 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:30:46,262 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:30:46,267 DEBUG HandlerThread:2651916 [system_info.py:__init__():26] System info init +2025-01-21 21:30:46,267 DEBUG HandlerThread:2651916 [system_info.py:__init__():41] System info init done +2025-01-21 21:30:46,267 INFO HandlerThread:2651916 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:30:46,267 INFO SystemMonitor:2651916 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:30:46,268 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:30:46,268 INFO HandlerThread:2651916 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:30:46,269 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:30:46,269 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:30:46,270 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:30:46,270 INFO SystemMonitor:2651916 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:30:46,281 DEBUG HandlerThread:2651916 [system_info.py:probe():152] Probing system +2025-01-21 21:30:46,283 DEBUG HandlerThread:2651916 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:30:46,288 DEBUG HandlerThread:2651916 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:30:46,288 DEBUG HandlerThread:2651916 [system_info.py:probe():200] Probing system done +2025-01-21 21:30:46,288 DEBUG HandlerThread:2651916 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:30:46.281548', 'startedAt': '2025-01-22T02:30:45.820874', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.7545, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 381.4694061279297}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:30:46,288 INFO HandlerThread:2651916 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:30:46,288 INFO HandlerThread:2651916 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:30:46,288 DEBUG HandlerThread:2651916 [system_info.py:_save_code():44] Saving code +2025-01-21 21:30:46,289 DEBUG HandlerThread:2651916 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:30:46,289 DEBUG HandlerThread:2651916 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:30:46,352 DEBUG HandlerThread:2651916 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:30:46,353 INFO HandlerThread:2651916 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:30:46,360 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:30:46,360 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:30:46,361 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:30:46,361 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:30:46,589 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:30:46,589 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:30:46,590 DEBUG SenderThread:2651916 [sender.py:send():379] send: telemetry +2025-01-21 21:30:46,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:46,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:30:46,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:30:46,633 INFO HandlerThread:2651916 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/tb" + save: true +} +control { + relay_id: "f6d28eb1c12a46838862bea7e051a09a" +} +_info { + stream_id: "pkbouru1" +} + +2025-01-21 21:30:46,633 DEBUG HandlerThread:2651916 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:30:46,652 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ea9nclmn-diff.patch +2025-01-21 21:30:46,658 INFO wandb-upload_1:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/f2w68x3r-code/FISH/eval_robot.py +2025-01-21 21:30:46,659 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/tpdhf4pz-wandb-metadata.json +2025-01-21 21:30:46,718 DEBUG SenderThread:2651916 [sender.py:send():379] send: tbrecord +2025-01-21 21:30:46,718 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:30:46,718 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 with policy live +2025-01-21 21:30:46,897 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/dfpwy5ze-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:30:47,169 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-metadata.json +2025-01-21 21:30:47,169 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/requirements.txt +2025-01-21 21:30:47,169 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:30:47,169 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code/FISH/eval_robot.py +2025-01-21 21:30:47,170 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/diff.patch +2025-01-21 21:30:47,170 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:30:47,170 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code/FISH +2025-01-21 21:30:47,170 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code +2025-01-21 21:30:47,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:48,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:49,168 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:30:49,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:50,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:51,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:51,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:52,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:53,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:54,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:55,016 DEBUG SenderThread:2651916 [sender.py:send():379] send: config +2025-01-21 21:30:55,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:56,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:30:56,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:57,172 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:30:57,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:58,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:30:59,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:00,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:01,173 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:01,589 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:31:01,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:31:01,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:01,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:02,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:03,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:04,593 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:05,174 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:05,593 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:06,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:07,175 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:07,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:07,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:08,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:09,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:10,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:11,176 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:11,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:12,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:12,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:13,594 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:14,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:15,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:16,589 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:31:16,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:31:16,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:17,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:17,719 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:18,177 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml +2025-01-21 21:31:18,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:19,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:20,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:21,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:22,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:23,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:23,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:24,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:25,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:26,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:27,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:28,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:28,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:29,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:30,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:31,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:31:31,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:31:31,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:32,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:33,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:33,677 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:34,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:35,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:36,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:37,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:38,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:39,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:39,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:40,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:41,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:42,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:43,184 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:43,609 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:44,573 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:31:44,574 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:31:44,574 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 with policy now +2025-01-21 21:31:44,575 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:31:44,575 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:31:44,575 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:31:44,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:44,833 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/83plspd5-media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos +2025-01-21 21:31:45,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:31:45,469 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/p2eq6pp7-media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 +2025-01-21 21:31:45,584 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:45,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:46,270 DEBUG SystemMonitor:2651916 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:31:46,271 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:31:46,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:31:46,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:31:46,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:47,185 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:47,599 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:48,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:49,186 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:49,600 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:50,600 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:50,953 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:51,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:52,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:53,187 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:53,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:54,188 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:31:54,489 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/oadx6w7s-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:31:54,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:55,188 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:55,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:56,025 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:31:56,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:57,189 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:57,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:58,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:31:59,190 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:31:59,601 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:00,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:01,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:32:01,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:32:01,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:01,705 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:02,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:03,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:04,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:05,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:06,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:07,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:07,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:08,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:09,604 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:10,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:11,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:12,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:12,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:13,604 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:14,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:15,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:16,271 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:32:16,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:32:16,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:32:16,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:17,195 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:17,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:17,662 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:18,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:19,196 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:19,605 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:20,606 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:21,606 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:22,606 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:23,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:23,606 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:24,608 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:25,607 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:26,607 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:27,607 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:28,592 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:28,607 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:29,608 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:30,608 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:31,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:32:31,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:32:31,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:32,608 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:33,611 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:33,714 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:33,757 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:32:33,757 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:32:33,758 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 with policy now +2025-01-21 21:32:33,758 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:32:33,758 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:32:33,759 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:32:33,981 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ig7c3sos-media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 +2025-01-21 21:32:34,201 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:32:34,201 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 +2025-01-21 21:32:34,201 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:32:34,484 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/mj0ozn2x-media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 +2025-01-21 21:32:34,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:35,201 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:35,615 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:36,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:37,618 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:38,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:39,203 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:39,417 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:39,621 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:40,615 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:41,203 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:41,615 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:42,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:43,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:44,593 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:44,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:45,618 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:46,272 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:32:46,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:32:46,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:32:46,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:47,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:48,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:49,063 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:32:49,063 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:32:49,063 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:32:49,206 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:32:49,206 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:32:49,618 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:50,537 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:50,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:51,206 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:51,617 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:52,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:53,207 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:53,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:54,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:55,208 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:32:55,596 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:32:55,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:56,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:57,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:58,619 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:32:59,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:00,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:01,590 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:33:01,590 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:33:01,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:01,710 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:02,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:03,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:04,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:05,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:06,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:07,593 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:07,620 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:08,628 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:08,984 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:33:08,984 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 with policy now +2025-01-21 21:33:08,991 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:33:08,991 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:33:08,992 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:33:08,992 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:33:09,198 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/zvcmr5ax-media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 +2025-01-21 21:33:09,212 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:09,213 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:33:09,213 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 +2025-01-21 21:33:09,213 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:33:09,477 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ty0mj2be-media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 +2025-01-21 21:33:09,631 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:10,625 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:11,213 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:11,627 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:12,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:12,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:13,627 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:14,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:15,214 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:15,625 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:16,272 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:33:16,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:33:16,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:33:16,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:17,215 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:17,627 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:17,720 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:18,628 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:19,628 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:20,628 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:21,627 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:22,632 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:23,595 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:23,636 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:24,636 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:25,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:26,637 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:27,219 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:33:27,469 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ol83ayog-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:33:27,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:27,887 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:33:27,887 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:33:27,887 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:33:28,219 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:33:28,646 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:28,993 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:29,219 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:29,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:30,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:31,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:33:31,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:33:31,639 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:32,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:33,221 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:33,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:34,639 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:34,732 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:35,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:36,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:37,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:38,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:39,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:40,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:40,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:41,640 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:42,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:43,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:44,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:45,598 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:45,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:46,273 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:33:46,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:33:46,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:33:46,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:47,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:48,641 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:49,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:50,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:50,742 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:51,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:52,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:53,227 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:33:53,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:54,642 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:55,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:56,437 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:33:56,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:57,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:58,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:33:59,643 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:00,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:01,591 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:34:01,591 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:34:01,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:01,744 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:02,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:03,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:04,644 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:05,653 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:06,650 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:06,761 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:34:06,761 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 with policy now +2025-01-21 21:34:06,761 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:34:06,763 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:06,764 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:34:06,765 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:34:06,770 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:34:06,990 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/x2vu43wd-media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 +2025-01-21 21:34:07,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:07,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:34:07,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 +2025-01-21 21:34:07,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:34:07,504 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/im2a57kj-media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 +2025-01-21 21:34:07,650 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:08,649 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:09,232 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:09,660 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:10,651 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:11,233 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:11,652 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:12,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:12,653 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:13,234 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:13,652 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:14,657 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:15,235 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:15,657 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:16,273 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:34:16,603 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:34:16,604 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:34:16,664 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:17,659 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:17,704 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:17,947 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:34:17,947 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:34:17,948 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:34:18,236 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:34:18,236 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:34:18,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:19,236 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:19,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:20,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:21,237 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:21,673 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:22,667 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:22,893 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:23,237 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:23,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:24,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:25,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:26,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:27,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:28,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:28,865 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:29,240 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:29,668 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:30,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:31,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:34:31,597 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:34:31,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:32,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:33,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:34,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:34,748 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:35,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:36,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:37,669 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:38,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:39,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:40,613 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:40,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:41,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:42,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:43,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:44,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:45,670 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:46,274 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:34:46,274 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:46,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:34:46,597 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:34:46,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:47,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:48,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:49,246 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:49,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:50,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:51,612 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:51,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:52,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:53,249 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:53,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:54,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:55,671 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:56,613 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:34:56,684 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:57,250 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:57,579 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:34:57,579 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 with policy now +2025-01-21 21:34:57,584 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:34:57,659 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:34:57,659 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:34:57,673 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:57,686 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:34:57,808 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/yj9tkrjw-media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 +2025-01-21 21:34:58,251 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:34:58,251 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 +2025-01-21 21:34:58,251 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval +2025-01-21 21:34:58,544 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/drleyppi-media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 +2025-01-21 21:34:58,675 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:34:59,251 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:34:59,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:00,683 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:01,602 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:35:01,603 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:35:01,666 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:01,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:02,677 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:03,253 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:03,684 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:04,679 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:05,254 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:05,682 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:06,678 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:07,633 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:07,684 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:08,686 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:09,693 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:10,021 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:35:10,021 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:10,022 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:10,238 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:35:10,238 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval +2025-01-21 21:35:10,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table +2025-01-21 21:35:10,398 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:35:10,442 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/6i5c7b10-media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json +2025-01-21 21:35:10,546 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/o6w4rf8n-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:10,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:10,748 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:35:10,782 INFO wandb-upload_1:2651916 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpaetffev_ +2025-01-21 21:35:10,793 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:35:11,207 INFO SenderThread:2651916 [sender.py:send_request_log_artifact():1525] logged artifact run-pkbouru1-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTA1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NzUyOQ==', 'latestArtifact': None}} +2025-01-21 21:35:11,207 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:35:11,207 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json with policy now +2025-01-21 21:35:11,207 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:35:11,208 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,208 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,208 DEBUG SenderThread:2651916 [sender.py:send():379] send: telemetry +2025-01-21 21:35:11,208 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,212 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,212 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,213 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,213 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,213 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,213 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,214 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,214 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,214 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,214 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,215 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,215 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,215 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,215 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,216 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,216 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,216 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,216 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,217 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,217 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:11,217 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:11,217 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:35:11,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:11,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:11,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:11,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json +2025-01-21 21:35:11,257 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval +2025-01-21 21:35:11,460 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/0rml6o24-media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json +2025-01-21 21:35:11,483 INFO wandb-upload_1:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/ce38462j-media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json +2025-01-21 21:35:11,604 INFO wandb-upload_2:2651916 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpuuksyuk_ +2025-01-21 21:35:11,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:11,999 INFO SenderThread:2651916 [sender.py:send_request_log_artifact():1525] logged artifact run-pkbouru1-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTA2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTE5NzUzMA==', 'latestArtifact': None}} +2025-01-21 21:35:11,999 DEBUG SenderThread:2651916 [sender.py:send():379] send: files +2025-01-21 21:35:11,999 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json with policy now +2025-01-21 21:35:11,999 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:35:12,000 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:12,000 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:12,249 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/hmyxkqwf-media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json +2025-01-21 21:35:12,256 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:12,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:13,001 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:13,257 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:13,680 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:14,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:15,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:16,274 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:35:16,597 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:35:16,597 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:35:16,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:17,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:18,665 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:18,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:19,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:20,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:21,681 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:22,682 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:23,682 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:24,616 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:24,682 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:25,409 DEBUG SenderThread:2651916 [sender.py:send():379] send: exit +2025-01-21 21:35:25,409 INFO SenderThread:2651916 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:35:25,410 INFO SenderThread:2651916 [sender.py:send_exit():588] handling runtime: 279 +2025-01-21 21:35:25,411 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:25,411 INFO SenderThread:2651916 [sender.py:send_exit():594] send defer +2025-01-21 21:35:25,411 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,411 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:35:25,411 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:25,411 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:35:25,411 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:35:25,411 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,411 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:35:25,412 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:25,412 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:35:25,412 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:35:25,412 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,412 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:35:25,412 INFO HandlerThread:2651916 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:35:25,412 DEBUG SystemMonitor:2651916 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:35:25,412 DEBUG SystemMonitor:2651916 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:35:25,413 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:35:25,413 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:35:25,485 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:35:25,485 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:35:25,486 INFO HandlerThread:2651916 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:35:25,486 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:25,486 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:35:25,486 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:35:25,486 DEBUG SenderThread:2651916 [sender.py:send():379] send: stats +2025-01-21 21:35:25,486 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,486 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:35:25,487 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:25,487 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:35:25,487 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:35:25,487 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:25,487 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:35:26,262 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:27,263 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:32,029 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:32,029 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:32,029 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:35:32,030 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:32,030 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:35:32,030 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:35:32,031 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:32,031 DEBUG SenderThread:2651916 [sender.py:send():379] send: history +2025-01-21 21:35:32,031 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:32,031 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:35:32,031 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:35:32,032 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:32,032 DEBUG SenderThread:2651916 [sender.py:send():379] send: summary +2025-01-21 21:35:32,036 INFO SenderThread:2651916 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:35:32,036 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:32,036 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:35:32,036 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:35:32,036 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:32,036 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:35:32,037 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:32,037 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:35:32,040 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:35:32,149 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:35:32,149 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:32,149 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:35:32,150 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:32,150 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:35:32,274 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml +2025-01-21 21:35:32,274 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:32,411 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:33,056 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:35:33,056 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:33,056 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:33,056 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:35:33,057 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:33,057 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:35:33,057 INFO SenderThread:2651916 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:35:33,059 INFO SenderThread:2651916 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:35:33,061 INFO SenderThread:2651916 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:35:33,076 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:35:33,077 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:33,077 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:35:33,077 DEBUG SenderThread:2651916 [sender.py:send():379] send: artifact +2025-01-21 21:35:33,274 INFO Thread-12 :2651916 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:33,411 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:33,535 INFO wandb-upload_0:2651916 [upload_job.py:push():85] Skipped uploading /tmp/tmpjb9r83j6/wandb-job.json +2025-01-21 21:35:33,535 INFO wandb-upload_1:2651916 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmphi4_s9v7 +2025-01-21 21:35:33,630 INFO wandb-upload_2:2651916 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmppc454fha +2025-01-21 21:35:34,442 INFO SenderThread:2651916 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3MzQyMw==', 'versionIndex': 455}}} +2025-01-21 21:35:34,442 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:34,442 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:35:34,442 INFO SenderThread:2651916 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:35:35,275 INFO SenderThread:2651916 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files +2025-01-21 21:35:35,275 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml config.yaml +2025-01-21 21:35:35,275 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json wandb-summary.json +2025-01-21 21:35:35,275 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/requirements.txt requirements.txt +2025-01-21 21:35:35,279 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:35:35,279 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:35,279 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log output.log +2025-01-21 21:35:35,281 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/diff.patch diff.patch +2025-01-21 21:35:35,281 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:35:35,283 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 media/videos/eval/2_eval_2_6e61939f3ab9157666d6.mp4 +2025-01-21 21:35:35,283 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 media/videos/eval/3_eval_3_cf7dc702773eb47d1f7e.mp4 +2025-01-21 21:35:35,283 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 media/videos/eval/1_eval_1_335a125e2c942caab269.mp4 +2025-01-21 21:35:35,284 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 media/videos/eval/4_eval_4_4fab844ed0fc1ded78b7.mp4 +2025-01-21 21:35:35,284 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 media/videos/eval/0_eval_0_666330ccf994f224efc7.mp4 +2025-01-21 21:35:35,284 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json media/table/eval/success_rate_ci_5_6f7b04d7609ef203f175.table.json +2025-01-21 21:35:35,285 INFO SenderThread:2651916 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json media/table/eval/total_success_rate_ci_6_24085c194cbcf9ae6df8.table.json +2025-01-21 21:35:35,285 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:35:35,285 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:35,285 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,285 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:35:35,285 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,285 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:35:35,285 INFO SenderThread:2651916 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:35:35,412 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:35,412 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:35,457 INFO wandb-upload_1:2651916 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/wandb-summary.json +2025-01-21 21:35:35,480 INFO wandb-upload_0:2651916 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/config.yaml +2025-01-21 21:35:35,502 INFO wandb-upload_2:2651916 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/requirements.txt +2025-01-21 21:35:35,505 INFO wandb-upload_4:2651916 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/files/output.log +2025-01-21 21:35:35,601 INFO wandb-upload_3:2651916 [upload_job.py:push():130] Uploaded file /tmp/tmp7bxqq5mnwandb/e2j3ddfv-events.out.tfevents.1737513046.leonmkim-ROG-Strix-G15CS-G15CS.2651732.0 +2025-01-21 21:35:35,801 INFO Thread-11 (_thread_body):2651916 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:35:35,801 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,801 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:35:35,802 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,802 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:35:35,802 INFO SenderThread:2651916 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:35:35,803 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:35:35,803 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,803 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:35:35,803 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,803 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:35:35,803 INFO SenderThread:2651916 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:35:35,872 INFO SenderThread:2651916 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:35:35,872 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:35:35,872 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,872 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:35:35,873 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,873 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:35:35,873 INFO SenderThread:2651916 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:35:35,873 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:35:35,873 INFO HandlerThread:2651916 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:35:35,873 DEBUG SenderThread:2651916 [sender.py:send():379] send: final +2025-01-21 21:35:35,873 DEBUG SenderThread:2651916 [sender.py:send():379] send: footer +2025-01-21 21:35:35,873 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: defer +2025-01-21 21:35:35,873 INFO SenderThread:2651916 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:35:35,874 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:35,874 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:35,875 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:35:35,875 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:35:35,875 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:35:35,875 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:35:35,877 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:35:35,877 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:35:35,925 DEBUG SenderThread:2651916 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:35:35,925 INFO MainThread:2651916 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:35:35,925 INFO MainThread:2651916 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:35:35,926 INFO MainThread:2651916 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:35:35,926 DEBUG HandlerThread:2651916 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:35:35,926 INFO HandlerThread:2651916 [handler.py:finish():882] shutting down handler +2025-01-21 21:35:36,875 INFO WriterThread:2651916 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/run-pkbouru1.wandb +2025-01-21 21:35:36,925 INFO SenderThread:2651916 [sender.py:finish():1615] shutting down sender +2025-01-21 21:35:36,925 INFO SenderThread:2651916 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:35:36,925 INFO SenderThread:2651916 [file_pusher.py:join():175] waiting for file pusher diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug.log b/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..43f7eec06327a97ef6bbb9b0ee8b87769c98e767 --- /dev/null +++ b/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Configure stats pid to 2651732 +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/settings +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:30:45,825 INFO MainThread:2651732 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug.log +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/wandb/run-20250121_213045-pkbouru1/logs/debug-internal.log +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_2', 'wandb_notes': '4295_2_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1'} +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:init():619] starting backend +2025-01-21 21:30:45,826 INFO MainThread:2651732 [wandb_init.py:init():623] setting up manager +2025-01-21 21:30:45,831 INFO MainThread:2651732 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:30:45,832 INFO MainThread:2651732 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:30:45,843 INFO MainThread:2651732 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:30:45,849 INFO MainThread:2651732 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:30:46,185 INFO MainThread:2651732 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:30:46,257 INFO MainThread:2651732 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:30:46,258 INFO MainThread:2651732 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:30:46,589 INFO MainThread:2651732 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:30:46,589 INFO MainThread:2651732 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:30:46,589 INFO MainThread:2651732 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:30:46,589 INFO MainThread:2651732 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:30:46,592 INFO MainThread:2651732 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:30:46,592 INFO MainThread:2651732 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/pkbouru1/tb, True +2025-01-21 21:30:55,015 INFO MainThread:2651732 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:35:36,927 WARNING MsgRouterThr:2651732 [router.py:message_loop():77] message_loop has been closed diff --git a/pkbouru1/wandb/run-20250121_213045-pkbouru1/run-pkbouru1.wandb b/pkbouru1/wandb/run-20250121_213045-pkbouru1/run-pkbouru1.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e9494eee624617c2679e668eb587a593781072a6 Binary files /dev/null and b/pkbouru1/wandb/run-20250121_213045-pkbouru1/run-pkbouru1.wandb differ diff --git a/q5zzaq6a/.hydra/config.yaml b/q5zzaq6a/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..836895e41a10194087484706499cf57e8cf16e48 --- /dev/null +++ b/q5zzaq6a/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/q5zzaq6a/.hydra/hydra.yaml b/q5zzaq6a/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8300d13285f71a3324a854f19c114fef292bbf6 --- /dev/null +++ b/q5zzaq6a/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/q5zzaq6a/.hydra/overrides.yaml b/q5zzaq6a/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/q5zzaq6a/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/q5zzaq6a/episode_rosbags/aligned_depth_to_color_K.npy b/q5zzaq6a/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/q5zzaq6a/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/q5zzaq6a/episode_rosbags/cam_tf_world.npy b/q5zzaq6a/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/q5zzaq6a/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/q5zzaq6a/episode_rosbags/color_K.npy b/q5zzaq6a/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/q5zzaq6a/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/q5zzaq6a/episode_rosbags/depth_K.npy b/q5zzaq6a/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/q5zzaq6a/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/q5zzaq6a/episode_rosbags/episode_1_2025-01-21-21-51-46.bag b/q5zzaq6a/episode_rosbags/episode_1_2025-01-21-21-51-46.bag new file mode 100644 index 0000000000000000000000000000000000000000..c433ea616950a49a147758c8a129298e762c7fd0 --- /dev/null +++ b/q5zzaq6a/episode_rosbags/episode_1_2025-01-21-21-51-46.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1da4a1de79b542d869c8175342d0873893d739ba29f24a30816dd4c25f4b0be +size 1414991640 diff --git a/q5zzaq6a/episode_rosbags/episode_2_2025-01-21-21-52-18.bag b/q5zzaq6a/episode_rosbags/episode_2_2025-01-21-21-52-18.bag new file mode 100644 index 0000000000000000000000000000000000000000..3f2f65ea2c49f241358b4b8525bf28b86c4abc29 --- /dev/null +++ b/q5zzaq6a/episode_rosbags/episode_2_2025-01-21-21-52-18.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:76ea0d2b1ae810868796fa923e6a47182abbf22bfdbccb8818dd769e10cbaa8e +size 1380325030 diff --git a/q5zzaq6a/eval_robot.log b/q5zzaq6a/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..b5a302381f930f1907b151dc6bc2600893d22d62 --- /dev/null +++ b/q5zzaq6a/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-21 21:50:26,963][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:597: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:50:26,966][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:533: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/q5zzaq6a/eval_video/0_eval.mp4 b/q5zzaq6a/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3454e7598f8d5b96d569450526d6e421037a41fa --- /dev/null +++ b/q5zzaq6a/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fe7dc19834ce97a254b129191444e9b17ade5ae7b5174a19c08b15735c34e1c +size 1499707 diff --git a/q5zzaq6a/eval_video/1_eval.mp4 b/q5zzaq6a/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d4f9ad9426e189d1aa1ea82d336dafde19c45c4a --- /dev/null +++ b/q5zzaq6a/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ec83cc214455bb6907bcec840e439d8671a9ecf9f0a5de34fe0523b1fd7f8ca +size 887612 diff --git a/q5zzaq6a/eval_video/2_eval.mp4 b/q5zzaq6a/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb1036fbdb53d7f8e8300a5583a8bbb6c62e5ddf --- /dev/null +++ b/q5zzaq6a/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7f6fc91409ff331f77f84c99c8b751214bd44ab548bf669f09152e2f9100932 +size 882777 diff --git a/q5zzaq6a/eval_video/3_eval.mp4 b/q5zzaq6a/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad66d8ada6355a8a8ab6d85b407484c6ede6f2cb --- /dev/null +++ b/q5zzaq6a/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba9216e7dc0717aa319731c23875bd8c3537250a8da5c69e79881eedd6d8f4ce +size 1480301 diff --git a/q5zzaq6a/eval_video/4_eval.mp4 b/q5zzaq6a/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a6ef09b5aeb962271961b4227bd31d55d9852c02 --- /dev/null +++ b/q5zzaq6a/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a207dd3be5225e41ea05d104ba941451b310f9aeb3a9ab69b908a80bcddb1de2 +size 1484379 diff --git a/q5zzaq6a/tb/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 b/q5zzaq6a/tb/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 new file mode 100644 index 0000000000000000000000000000000000000000..9f11927e69f137cb05e475683d160d91b4657df6 --- /dev/null +++ b/q5zzaq6a/tb/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2cbc54bb3d623a06cb003d26cac5ed9004bb9ce059bf968a55abe7002e47fe6 +size 6539 diff --git a/q5zzaq6a/wandb/debug-internal.log b/q5zzaq6a/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5d2ba0eaeca8c4c36ad87c17ac52ad55fa42e78e --- /dev/null +++ b/q5zzaq6a/wandb/debug-internal.log @@ -0,0 +1,758 @@ +2025-01-21 21:50:32,700 INFO StreamThr :2657635 [internal.py:wandb_internal():85] W&B internal server running at pid: 2657635, started at: 2025-01-21 21:50:32.700411 +2025-01-21 21:50:32,701 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:50:32,713 INFO WriterThread:2657635 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/run-q5zzaq6a.wandb +2025-01-21 21:50:32,713 DEBUG SenderThread:2657635 [sender.py:send():379] send: header +2025-01-21 21:50:32,722 DEBUG SenderThread:2657635 [sender.py:send():379] send: run +2025-01-21 21:50:33,001 INFO SenderThread:2657635 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files +2025-01-21 21:50:33,001 INFO SenderThread:2657635 [sender.py:_start_run_threads():1188] run started: q5zzaq6a with start time 1737514232.701994 +2025-01-21 21:50:33,014 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:50:33,015 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:50:33,085 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:50:33,090 DEBUG HandlerThread:2657635 [system_info.py:__init__():26] System info init +2025-01-21 21:50:33,090 DEBUG HandlerThread:2657635 [system_info.py:__init__():41] System info init done +2025-01-21 21:50:33,090 INFO HandlerThread:2657635 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:50:33,090 INFO SystemMonitor:2657635 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:50:33,090 INFO HandlerThread:2657635 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:50:33,091 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:50:33,091 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:50:33,092 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:50:33,093 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:50:33,094 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:50:33,104 DEBUG HandlerThread:2657635 [system_info.py:probe():152] Probing system +2025-01-21 21:50:33,106 DEBUG HandlerThread:2657635 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:50:33,110 DEBUG HandlerThread:2657635 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:50:33,110 DEBUG HandlerThread:2657635 [system_info.py:probe():200] Probing system done +2025-01-21 21:50:33,111 DEBUG HandlerThread:2657635 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:50:33.104693', 'startedAt': '2025-01-22T02:50:32.686458', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.467375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.963, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.915, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 418.30762100219727}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:50:33,111 INFO HandlerThread:2657635 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:50:33,111 INFO HandlerThread:2657635 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:50:33,111 DEBUG HandlerThread:2657635 [system_info.py:_save_code():44] Saving code +2025-01-21 21:50:33,112 DEBUG HandlerThread:2657635 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:50:33,112 DEBUG HandlerThread:2657635 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:50:33,174 DEBUG HandlerThread:2657635 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:50:33,174 INFO HandlerThread:2657635 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:50:33,177 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:50:33,177 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:50:33,178 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:50:33,178 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:50:33,410 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:50:33,410 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:50:33,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:50:33,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:33,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:50:33,423 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/80eig8ty-wandb-metadata.json +2025-01-21 21:50:33,457 INFO HandlerThread:2657635 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/tb" + save: true +} +control { + relay_id: "c13fbe6decbf490f88545b861eb62ce0" +} +_info { + stream_id: "q5zzaq6a" +} + +2025-01-21 21:50:33,457 DEBUG HandlerThread:2657635 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:50:33,467 INFO wandb-upload_2:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/83g0v4to-diff.patch +2025-01-21 21:50:33,472 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/cyxgueju-code/FISH/eval_robot.py +2025-01-21 21:50:33,562 DEBUG SenderThread:2657635 [sender.py:send():379] send: telemetry +2025-01-21 21:50:33,562 DEBUG SenderThread:2657635 [sender.py:send():379] send: tbrecord +2025-01-21 21:50:33,563 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:50:33,563 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 with policy live +2025-01-21 21:50:33,748 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/m119crax-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:50:34,003 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:50:34,003 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/diff.patch +2025-01-21 21:50:34,003 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/requirements.txt +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-metadata.json +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code/FISH/eval_robot.py +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code/FISH +2025-01-21 21:50:34,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:35,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:36,002 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:36,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:36,856 DEBUG SenderThread:2657635 [sender.py:send():379] send: config +2025-01-21 21:50:37,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:37,870 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:38,003 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:38,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:39,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:40,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:40,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:41,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:42,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:42,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:43,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:43,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:44,005 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:44,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:45,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:46,006 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:46,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:47,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:48,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:50:48,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:50:48,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:48,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:49,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:50,007 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:50,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:51,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:52,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:53,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:54,009 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:54,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:54,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:55,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:56,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:57,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:58,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:59,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:59,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:00,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:01,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:02,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:03,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:51:03,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:51:03,457 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:04,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:04,498 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:05,012 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml +2025-01-21 21:51:05,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:06,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:07,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:08,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:09,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:10,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:10,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:11,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:12,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:13,416 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:14,417 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:15,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:15,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:16,417 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:17,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:18,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:51:18,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:51:18,453 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:19,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:20,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:20,563 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:21,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:22,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:23,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:24,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:25,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:26,197 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:26,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:27,130 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:51:27,130 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 with policy now +2025-01-21 21:51:27,131 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:51:27,132 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:51:27,132 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:51:27,133 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:51:27,405 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/4ha7vfr4-media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 +2025-01-21 21:51:27,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:28,022 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:28,022 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 +2025-01-21 21:51:28,022 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:51:28,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:51:28,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos +2025-01-21 21:51:28,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media +2025-01-21 21:51:28,350 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/t3wrlr1c-media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 +2025-01-21 21:51:28,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:29,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:30,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:30,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:31,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:31,793 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:32,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:32,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:33,094 DEBUG SystemMonitor:2657635 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:51:33,095 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:51:33,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:51:33,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:51:33,453 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:34,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:35,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:36,025 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:36,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:37,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:37,532 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:38,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:39,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:40,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:41,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:42,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:42,583 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:43,027 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:51:43,294 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/8dp9bw6f-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:51:43,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:44,027 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:44,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:45,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:46,028 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:46,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:47,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:48,029 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:48,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:51:48,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:51:48,457 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:48,568 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:49,422 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:50,422 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:51,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:52,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:53,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:54,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:54,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:55,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:56,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:57,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:58,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:59,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:59,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:00,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:01,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:02,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:03,104 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:52:03,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:52:03,412 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:52:03,453 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:04,425 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:04,547 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:05,461 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:06,425 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:06,747 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:52:06,747 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 with policy now +2025-01-21 21:52:06,747 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:52:06,748 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:52:06,748 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:52:06,748 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:52:06,959 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/k6evo0up-media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 +2025-01-21 21:52:07,035 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:52:07,035 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 +2025-01-21 21:52:07,035 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:52:07,338 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/04wk10ls-media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 +2025-01-21 21:52:07,426 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:08,035 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:08,427 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:09,036 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:09,433 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:10,165 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:10,430 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:11,430 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:12,037 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:12,430 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:13,428 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:14,038 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:14,428 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:14,848 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:52:14,849 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:52:14,849 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:52:15,038 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:52:15,038 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:52:15,428 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:15,549 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:16,038 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:16,429 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:17,428 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:18,039 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:18,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:52:18,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:52:18,440 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:19,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:20,040 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:20,440 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:20,557 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:21,441 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:22,441 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:23,441 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:24,441 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:25,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:25,877 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:26,043 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:26,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:27,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:28,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:29,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:30,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:31,416 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:31,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:32,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:33,095 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:52:33,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:52:33,412 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:52:33,453 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:34,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:35,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:36,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:36,525 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:37,449 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:38,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:38,255 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:52:38,255 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 with policy now +2025-01-21 21:52:38,257 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:52:38,258 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:52:38,258 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:52:38,259 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:52:38,445 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:38,485 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/81r70su4-media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 +2025-01-21 21:52:39,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:52:39,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:39,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 +2025-01-21 21:52:39,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:52:39,335 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/fuaftwst-media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 +2025-01-21 21:52:39,451 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:40,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:40,450 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:41,048 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:41,456 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:41,686 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:42,460 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:43,455 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:44,049 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:44,461 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:45,455 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:46,049 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:46,459 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:47,461 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:47,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:48,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:52:48,414 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:52:48,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:49,051 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:52:49,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:49,672 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:52:49,672 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:52:49,690 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:52:49,810 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/kxiyxen9-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:52:50,051 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:52:50,051 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:50,468 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:51,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:52,052 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:52,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:52,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:53,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:54,052 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:54,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:55,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:56,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:57,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:58,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:58,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:59,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:00,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:01,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:02,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:03,096 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:53:03,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:53:03,414 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:53:03,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:03,573 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:04,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:05,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:06,465 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:07,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:08,057 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:08,465 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:09,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:09,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:10,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:11,465 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:12,465 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:13,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:14,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:14,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:15,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:16,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:17,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:18,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:53:18,414 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:53:18,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:19,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:19,529 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:20,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:21,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:22,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:23,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:24,468 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:25,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:25,468 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:26,468 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:27,469 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:28,066 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:28,214 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:53:28,214 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 with policy now +2025-01-21 21:53:28,218 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:53:28,218 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:53:28,218 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:53:28,233 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:53:28,463 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/pa105z2x-media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 +2025-01-21 21:53:28,469 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:29,064 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:53:29,064 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 +2025-01-21 21:53:29,064 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:53:29,366 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/ykc075sz-media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 +2025-01-21 21:53:29,477 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:30,064 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:30,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:30,474 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:31,477 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:32,479 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:33,096 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:53:33,416 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:53:33,417 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:53:33,475 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:34,066 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:34,475 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:35,475 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:35,689 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:36,066 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:36,478 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:37,477 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:38,479 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:39,482 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:39,732 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:53:39,732 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:53:39,733 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:53:40,068 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:53:40,068 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:53:40,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:41,480 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:41,661 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:42,069 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:42,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:43,482 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:44,480 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:45,480 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:46,070 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:46,480 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:46,812 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:47,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:48,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:53:48,415 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:53:48,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:49,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:50,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:51,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:52,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:52,625 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:53,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:54,073 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:54,482 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:55,483 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:56,483 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:57,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:58,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:58,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:59,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:00,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:01,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:02,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:03,097 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:54:03,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:54:03,415 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:54:03,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:03,501 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:04,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:05,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:06,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:07,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:08,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:09,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:09,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:10,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:11,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:12,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:13,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:14,425 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:14,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:15,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:16,487 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:17,487 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:18,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:54:18,415 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:54:18,487 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:19,426 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:19,469 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:54:19,469 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 with policy now +2025-01-21 21:54:19,474 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:54:19,475 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:19,475 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:19,475 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:19,489 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:19,752 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/n2bj6gp6-media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 +2025-01-21 21:54:20,082 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:20,082 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:20,082 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 +2025-01-21 21:54:20,082 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:54:20,389 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/ayptpw1r-media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 +2025-01-21 21:54:20,490 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:21,497 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:22,083 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:22,499 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:23,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:24,083 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:24,431 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:24,506 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:25,500 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:26,501 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:27,499 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:28,085 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:28,515 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:29,436 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:29,512 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:30,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:31,506 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:32,507 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:33,103 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:54:33,422 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:54:33,422 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:54:33,506 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:34,454 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:34,501 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:34,799 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:34,799 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:34,800 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:35,087 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:35,087 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:35,292 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:54:35,293 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:54:35,346 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/dc1eu72r-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:35,476 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:54:35,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:35,826 INFO wandb-upload_0:2657635 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp72xx187r +2025-01-21 21:54:35,862 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:54:35,870 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval +2025-01-21 21:54:36,245 INFO SenderThread:2657635 [sender.py:send_request_log_artifact():1525] logged artifact run-q5zzaq6a-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcwMTgwMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMTQxMA==', 'latestArtifact': None}} +2025-01-21 21:54:36,245 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:54:36,245 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json with policy now +2025-01-21 21:54:36,245 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:36,245 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,246 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,246 DEBUG SenderThread:2657635 [sender.py:send():379] send: telemetry +2025-01-21 21:54:36,246 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,250 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,250 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,250 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,250 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,251 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,251 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,251 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,251 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,252 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,252 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,252 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,252 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,253 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,253 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,253 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,253 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,254 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,254 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,255 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,255 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,255 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,255 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:54:36,436 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/104pge0x-media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json +2025-01-21 21:54:36,466 INFO wandb-upload_2:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/rkbjsa9i-media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json +2025-01-21 21:54:36,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:36,680 INFO wandb-upload_0:2657635 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpivxe_ti9 +2025-01-21 21:54:36,739 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/0ukvjqy0-media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json +2025-01-21 21:54:37,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:37,116 INFO SenderThread:2657635 [sender.py:send_request_log_artifact():1525] logged artifact run-q5zzaq6a-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcwMTgyMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMTQxMQ==', 'latestArtifact': None}} +2025-01-21 21:54:37,117 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:54:37,117 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json with policy now +2025-01-21 21:54:37,117 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:37,117 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:37,118 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:37,389 INFO wandb-upload_2:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/j03x1hmm-media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json +2025-01-21 21:54:37,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:38,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:38,089 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:38,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:39,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:40,431 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:40,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:41,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:42,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:43,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:44,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:45,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:46,431 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:46,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:47,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:48,416 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:54:48,416 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:54:48,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:49,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:50,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:51,472 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:51,613 DEBUG SenderThread:2657635 [sender.py:send():379] send: exit +2025-01-21 21:54:51,613 INFO SenderThread:2657635 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:54:51,613 INFO SenderThread:2657635 [sender.py:send_exit():588] handling runtime: 258 +2025-01-21 21:54:51,613 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:send_exit():594] send defer +2025-01-21 21:54:51,614 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,614 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:54:51,614 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:54:51,614 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,614 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:54:51,614 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:54:51,615 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,615 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:54:51,615 INFO HandlerThread:2657635 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:54:51,615 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:54:51,615 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:54:51,615 DEBUG SystemMonitor:2657635 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:54:51,616 DEBUG SystemMonitor:2657635 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:54:51,667 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:54:51,667 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:54:51,667 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:54:51,668 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:51,668 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:54:51,668 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:54:51,668 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:54:51,668 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,668 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:54:51,668 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:51,669 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:54:51,669 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:54:51,669 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,669 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:54:52,094 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml +2025-01-21 21:54:52,094 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:54,095 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:57,804 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:57,804 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:57,804 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:54:57,805 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:57,805 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:54:57,805 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:54:57,805 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:57,806 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:57,806 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:57,806 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:57,806 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:54:57,806 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:57,807 DEBUG SenderThread:2657635 [sender.py:send():379] send: summary +2025-01-21 21:54:57,808 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:57,808 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:57,808 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:54:57,809 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:54:57,809 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:57,809 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:54:57,809 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:57,809 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:54:57,809 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:54:57,810 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:57,810 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:57,810 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:54:57,810 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:57,810 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:54:58,096 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:58,471 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:58,855 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:54:58,855 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:58,856 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:58,856 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:54:58,856 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:58,856 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:54:58,856 INFO SenderThread:2657635 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:54:58,859 INFO SenderThread:2657635 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:54:58,860 INFO SenderThread:2657635 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:54:58,870 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:54:58,870 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:58,870 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:54:58,871 DEBUG SenderThread:2657635 [sender.py:send():379] send: artifact +2025-01-21 21:54:59,096 INFO SenderThread:2657635 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 21:54:59,096 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,096 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:54:59,096 INFO SenderThread:2657635 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:54:59,097 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:59,097 INFO SenderThread:2657635 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files +2025-01-21 21:54:59,097 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml config.yaml +2025-01-21 21:54:59,097 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json wandb-summary.json +2025-01-21 21:54:59,098 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/requirements.txt requirements.txt +2025-01-21 21:54:59,098 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:54:59,098 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:59,098 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log output.log +2025-01-21 21:54:59,100 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/diff.patch diff.patch +2025-01-21 21:54:59,103 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:54:59,107 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,107 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:54:59,107 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:54:59,295 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:59,299 INFO wandb-upload_2:2657635 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/requirements.txt +2025-01-21 21:54:59,347 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml +2025-01-21 21:54:59,353 INFO wandb-upload_4:2657635 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:59,418 INFO wandb-upload_3:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/691vlpdl-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:59,471 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:59,471 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:59,619 INFO Thread-11 (_thread_body):2657635 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:54:59,619 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,619 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:54:59,619 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,619 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:54:59,619 INFO SenderThread:2657635 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:54:59,620 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:54:59,620 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,620 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:54:59,621 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,621 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:54:59,621 INFO SenderThread:2657635 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:54:59,697 INFO SenderThread:2657635 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:54:59,697 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:54:59,698 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,698 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:54:59,698 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,698 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:54:59,698 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:54:59,698 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,698 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:54:59,698 DEBUG SenderThread:2657635 [sender.py:send():379] send: final +2025-01-21 21:54:59,699 DEBUG SenderThread:2657635 [sender.py:send():379] send: footer +2025-01-21 21:54:59,699 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,699 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:54:59,699 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:59,699 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:59,700 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:54:59,700 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:59,700 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:54:59,700 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:54:59,702 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:54:59,703 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:59,758 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:59,758 INFO MainThread:2657635 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:54:59,759 INFO MainThread:2657635 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:54:59,759 INFO MainThread:2657635 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:54:59,759 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:54:59,759 INFO HandlerThread:2657635 [handler.py:finish():882] shutting down handler +2025-01-21 21:55:00,700 INFO WriterThread:2657635 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/run-q5zzaq6a.wandb +2025-01-21 21:55:00,758 INFO SenderThread:2657635 [sender.py:finish():1615] shutting down sender +2025-01-21 21:55:00,758 INFO SenderThread:2657635 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:55:00,759 INFO SenderThread:2657635 [file_pusher.py:join():175] waiting for file pusher diff --git a/q5zzaq6a/wandb/debug.log b/q5zzaq6a/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f026f0f970acfff54624956d75308a4a4c218fb1 --- /dev/null +++ b/q5zzaq6a/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Configure stats pid to 2657441 +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/settings +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug.log +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug-internal.log +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_2', 'wandb_notes': '4295_2_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a'} +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:init():619] starting backend +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:init():623] setting up manager +2025-01-21 21:50:32,700 INFO MainThread:2657441 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:50:32,701 INFO MainThread:2657441 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:50:32,712 INFO MainThread:2657441 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:50:32,719 INFO MainThread:2657441 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:50:33,014 INFO MainThread:2657441 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:50:33,080 INFO MainThread:2657441 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:50:33,080 INFO MainThread:2657441 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:50:33,411 INFO MainThread:2657441 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:50:33,411 INFO MainThread:2657441 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:50:33,411 INFO MainThread:2657441 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:50:33,411 INFO MainThread:2657441 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:50:33,412 INFO MainThread:2657441 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:50:33,412 INFO MainThread:2657441 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/tb, True +2025-01-21 21:50:36,855 INFO MainThread:2657441 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:55:00,760 WARNING MsgRouterThr:2657441 [router.py:message_loop():77] message_loop has been closed diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code/FISH/eval_robot.py b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..905b7696ca4098dca24a4e9f941df06e4f0969c1 --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code/FISH/eval_robot.py @@ -0,0 +1,606 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config), + # hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + # self.cfg.agent.config.policy_cfg.input_shapes, + # hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af39a90268173dfeb9dddd9ab687cfa48311800e --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml @@ -0,0 +1,936 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 2 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '4295_2' +wandb_notes: + desc: null + value: 4295_2_restarted_1 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: false +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 7 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +restore_missing_checkpoint: + desc: null + value: null +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 2 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + crop_distractors_tlhw: + - 32 + - 68 + - 174 + - 174 + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +test: + desc: null + value: + username: leonmkim + hostname: grasp-login1 + name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + root_dir: ~/fish_leon + expert_dataset_dirpath: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + expert_dataset: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + semantic_demo_grouping_name: semantic_demo_grouping.yaml + semantic_demo_grouping: ~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml + include_groups_list: all + batch_size: 128 + store_dataset_in_memory: false + use_tb: true + local_snapshot_root_dir: /mnt/bighdd/fish_contact_backup + local_snapshot_dir: /mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1 + resume_wandb_run: false +feature_type: + desc: null + value: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 7500 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1737514232 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/diff.patch b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..d3a5568bb3f03cd705321e234cb5e34a755166b7 --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/diff.patch @@ -0,0 +1,25 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index bb5fa99..c37123f 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -82,7 +82,7 @@ contact_estimation_model_ckpt_path: '~/fish_leon/contact_estimation/artifacts/17 + + # Number of evaluation trajectories + # num_eval: 10 +-num_eval: 20 ++num_eval: 5 + + debug_timestamps: False + open_loop: False +@@ -144,9 +144,9 @@ load_checkpoint: ${agent.load_checkpoint} + # wandb_run_id: '3465_2' # seed 2 + + # crp_D+mask+contact+act history +-wandb_run_id: '4295_0' ++# wandb_run_id: '4295_0' + # wandb_run_id: '4295_1' # seed 1 +-# wandb_run_id: '4295_2' # seed 2 ++wandb_run_id: '4295_2' # seed 2 + + # RGBD+mask+act history + # wandb_run_id: '1009_1' # seed 1 diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..88f41bdcb2cf2d65fe1eba7609c384375c388041 --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.14516133077088833, 0.9549263416810942, 0.18293228751801865, 0.9330929740397204, 0.2852150500250279, 0.8864275961341479, 0.26130985473399826, 0.8173232524952435]]} \ No newline at end of file diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json new file mode 100644 index 0000000000000000000000000000000000000000..242c46e4c12228518e77db4ebc81e9195aa557ec --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[7, 10, 0.7, 0.38132041695107355, 0.940909355790648, 0.42515620590753167, 0.9237295090304732, 0.43590326608509267, 0.8963951020640086, 0.4820855259931154, 0.8360364883466034]]} \ No newline at end of file diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3454e7598f8d5b96d569450526d6e421037a41fa --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fe7dc19834ce97a254b129191444e9b17ade5ae7b5174a19c08b15735c34e1c +size 1499707 diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d4f9ad9426e189d1aa1ea82d336dafde19c45c4a --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ec83cc214455bb6907bcec840e439d8671a9ecf9f0a5de34fe0523b1fd7f8ca +size 887612 diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..bb1036fbdb53d7f8e8300a5583a8bbb6c62e5ddf --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7f6fc91409ff331f77f84c99c8b751214bd44ab548bf669f09152e2f9100932 +size 882777 diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ad66d8ada6355a8a8ab6d85b407484c6ede6f2cb --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ba9216e7dc0717aa319731c23875bd8c3537250a8da5c69e79881eedd6d8f4ce +size 1480301 diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a6ef09b5aeb962271961b4227bd31d55d9852c02 --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a207dd3be5225e41ea05d104ba941451b310f9aeb3a9ab69b908a80bcddb1de2 +size 1484379 diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a82ffb56420274f538fe3c3c2f4faac7600c7913 --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log @@ -0,0 +1,274 @@ + +loaded agent with feature_type: 180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1737514236.857232]: resetting environment +[INFO] [1737514236.866253]: cleared current plan +[INFO] [1737514236.867104]: moving to home +[INFO] [1737514237.869327]: reached home +[INFO] [1737514237.871244]: reset action history +[INFO] [1737514240.580376]: environment reset +Starting episode 0 +[INFO] [1737514240.584527]: resetting environment +[INFO] [1737514240.593070]: cleared current plan +[INFO] [1737514240.594739]: moving to home +[INFO] [1737514241.597337]: reached home +[INFO] [1737514241.598693]: reset action history +[INFO] [1737514244.310145]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1737514247.559953]: resetting environment +[INFO] [1737514247.564150]: cleared current plan +[INFO] [1737514247.564960]: moving to home +[INFO] [1737514248.567200]: reached home +[INFO] [1737514248.568496]: reset action history +[INFO] [1737514251.281543]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/episode_rosbags/episode_0_2025-01-21-21-50-51.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2657441.log +started roslaunch server http://158.130.50.37:39615/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2657741] +process[print_text-2]: started with pid [2657765] +process[rosbag_record-3]: started with pid [2657766] +started bagging! +[INFO] [1737514286.196591]: Storing episode... +[rosbag_record-3] killing on exit +[pub_text-1] killing on exit +[print_text-2] killing on exit +[INFO] [1737514286.926467]: Stored episode 1. +[INFO] [1737514286.927094]: Saving video... +[INFO] [1737514287.119287]: Video saved! +Starting episode 1 +[INFO] [1737514287.132514]: resetting environment +[INFO] [1737514287.142304]: cleared current plan +[INFO] [1737514287.142593]: moving to home +[INFO] [1737514290.745014]: reached home +[INFO] [1737514290.745569]: reset action history +[INFO] [1737514293.454881]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514302.587146]: resetting environment +[INFO] [1737514302.590934]: cleared current plan +[INFO] [1737514302.591108]: moving to home +[INFO] [1737514303.592690]: reached home +[INFO] [1737514303.593086]: reset action history +[INFO] [1737514306.302274]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/episode_rosbags/episode_1_2025-01-21-21-51-46.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2657441.log +started roslaunch server http://158.130.50.37:32911/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2657892] +process[print_text-5]: started with pid [2657893] +process[rosbag_record-6]: started with pid [2657894] +started bagging! +preempting episode +[INFO] [1737514325.420720]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1737514326.229526]: Stored episode 2. +[INFO] [1737514326.230095]: Saving video... +[INFO] [1737514326.720223]: Video saved! +Starting episode 2 +[INFO] [1737514326.749980]: resetting environment +[INFO] [1737514326.760374]: cleared current plan +[INFO] [1737514326.760629]: moving to home +[WARN] [1737514326.674402]: Plan exhausted +[WARN] [1737514326.714497]: Plan exhausted +[WARN] [1737514326.754364]: Plan exhausted +[INFO] [1737514330.163984]: reached home +[INFO] [1737514330.168381]: reset action history +[INFO] [1737514332.946541]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514334.533478]: resetting environment +[INFO] [1737514334.546931]: cleared current plan +[INFO] [1737514334.547223]: moving to home +[INFO] [1737514335.549271]: reached home +[INFO] [1737514335.550633]: reset action history +[INFO] [1737514338.259392]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/episode_rosbags/episode_2_2025-01-21-21-52-18.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2657441.log +started roslaunch server http://158.130.50.37:38737/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2658034] +process[print_text-8]: started with pid [2658035] +process[rosbag_record-9]: started with pid [2658036] +started bagging! +For topic gripper_width: timestamp difference is -239068470 for nearest: 1737514344512500086 - target: 1737514344751568556 at idx 88 +preempting episode +[INFO] [1737514356.957541]: Storing episode... +[pub_text-7] killing on exit +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[WARN] [1737514358.140262]: Plan exhausted +[WARN] [1737514358.177402]: Plan exhausted +[WARN] [1737514358.218225]: Plan exhausted +[INFO] [1737514357.761391]: Stored episode 3. +[INFO] [1737514357.762650]: Saving video... +[INFO] [1737514358.223465]: Video saved! +Starting episode 3 +[INFO] [1737514358.258312]: resetting environment +[INFO] [1737514358.279441]: cleared current plan +[INFO] [1737514358.279689]: moving to home +[WARN] [1737514358.260988]: Plan exhausted +[INFO] [1737514361.685169]: reached home +[INFO] [1737514361.689780]: reset action history +[INFO] [1737514364.413465]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514368.714143]: resetting environment +[INFO] [1737514368.733612]: cleared current plan +[INFO] [1737514368.733839]: moving to home +[INFO] [1737514369.737006]: reached home +[INFO] [1737514369.741016]: reset action history +[INFO] [1737514372.465566]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/episode_rosbags/episode_3_2025-01-21-21-52-52.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2657441.log +started roslaunch server http://158.130.50.37:41869/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2658179] +process[print_text-11]: started with pid [2658180] +process[rosbag_record-12]: started with pid [2658204] +started bagging! +For topic gripper_width: timestamp difference is -201495724 for nearest: 1737514386112475080 - target: 1737514386313970804 at idx 178 +[INFO] [1737514407.120914]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1737514407.882586]: Stored episode 4. +[INFO] [1737514407.886040]: Saving video... +[INFO] [1737514408.174807]: Video saved! +Starting episode 4 +[INFO] [1737514408.224396]: resetting environment +[INFO] [1737514408.247188]: cleared current plan +[INFO] [1737514408.260019]: moving to home +[INFO] [1737514411.872805]: reached home +[INFO] [1737514411.880650]: reset action history +[INFO] [1737514414.641546]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1737514419.574433]: resetting environment +[INFO] [1737514419.605405]: cleared current plan +[INFO] [1737514419.614449]: moving to home +[INFO] [1737514420.620235]: reached home +[INFO] [1737514420.623985]: reset action history +[INFO] [1737514423.358403]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/episode_rosbags/episode_4_2025-01-21-21-53-43.bag +... logging to /home/leonmkim/.ros/log/1b77c894-d3af-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2657441.log +started roslaunch server http://158.130.50.37:45577/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2658344] +process[print_text-14]: started with pid [2658345] +process[rosbag_record-15]: started with pid [2658346] +started bagging! +For topic gripper_width: timestamp difference is 105391991 for nearest: 1737514431579143298 - target: 1737514431473751307 at idx 113 +For topic gripper_width: timestamp difference is 107945429 for nearest: 1737514432879127012 - target: 1737514432771181583 at idx 130 +[INFO] [1737514458.362433]: Storing episode... +[pub_text-13] killing on exit +[print_text-14] killing on exit[rosbag_record-15] killing on exit +[INFO] [1737514459.181932]: Stored episode 5. +[INFO] [1737514459.197960]: Saving video... +[INFO] [1737514459.425507]: Video saved! +[INFO] [1737514459.474698]: resetting environment +[INFO] [1737514459.495092]: cleared current plan +[INFO] [1737514459.495675]: moving to home +[INFO] [1737514463.099311]: reached home +[INFO] [1737514463.102778]: reset action history +[INFO] [1737514465.893958]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/requirements.txt b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-metadata.json b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d401bd755457fa02d9918ef5a32875cf9f5aa9ec --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-22T02:50:33.104693", + "startedAt": "2025-01-22T02:50:32.686458", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "dc9da9fa0ef2996c7fec8d105e7933e728cd952f" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.467375, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 4499.963, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.915, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 418.30762100219727 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e80270d98cc27f7af23f63d076c9978bfc783a10 --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "7fe7dc19834ce97a254b129191444e9b17ade5ae7b5174a19c08b15735c34e1c", "size": 1499707, "path": "media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4"}, "global_step": 541, "_timestamp": 1737514475.77997, "_runtime": 243.07797598838806, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "4ec83cc214455bb6907bcec840e439d8671a9ecf9f0a5de34fe0523b1fd7f8ca", "size": 887612, "path": "media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4"}, "eval/num_success": 3.0, "episode": 10.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.14516133069992065, "eval/success_rate_ub_0.01": 0.9549263119697571, "eval/success_rate_lb_0.025": 0.1829322874546051, "eval/success_rate_ub_0.025": 0.9330929517745972, "eval/success_rate_lb_0.05": 0.2852150499820709, "eval/success_rate_ub_0.05": 0.8864275813102722, "eval/success_rate_lb_0.1": 0.2613098621368408, "eval/success_rate_ub_0.1": 0.8173232674598694, "eval/2_eval": {"_type": "video-file", "sha256": "e7f6fc91409ff331f77f84c99c8b751214bd44ab548bf669f09152e2f9100932", "size": 882777, "path": "media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "ba9216e7dc0717aa319731c23875bd8c3537250a8da5c69e79881eedd6d8f4ce", "size": 1480301, "path": "media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "a207dd3be5225e41ea05d104ba941451b310f9aeb3a9ab69b908a80bcddb1de2", "size": 1484379, "path": "media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "c77850ebfe62f375339bb95d11251517860ee80f4b166b49083f2901d1469cc1", "size": 401, "artifact_path": "wandb-client-artifact://9v8u9q5ggiczqy822wx5p7sqgnzre448mhejgb2g27xjhw4oruzbb01jiqxsqg5i8uuzsiwmxnmfj1vomuw1q4057ervhcbtqugpalc7c0nt4oaqgvsitjeka7mkf0an/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://p53h1ci1jna2lfm577x5s5lq7vsed5n9rs0el337q3nnx2cboocp73suv12hxlk748shf1rid30lij733yxolwb3k7cye10kwp43aegx5hjr69cof8i6kdgl55keruwx:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 7, "total_num_episodes": 10, "total_success_rate": 0.7, "total_success_rate_lb_0.01": 0.38132041695107355, "total_success_rate_ub_0.01": 0.940909355790648, "total_success_rate_lb_0.025": 0.42515620590753167, "total_success_rate_ub_0.025": 0.9237295090304732, "total_success_rate_lb_0.05": 0.43590326608509267, "total_success_rate_ub_0.05": 0.8963951020640086, "total_success_rate_lb_0.1": 0.4820855259931154, "total_success_rate_ub_0.1": 0.8360364883466034, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "36a4b75cedb5a99e4ccbc9234e65c1b9fe7289ef9c541a8695f5358043cfbb0e", "size": 506, "artifact_path": "wandb-client-artifact://hwwcjq4o1em0hp8wix4gy7ri5ej59z7akj4xasyqj32xn9uz4rg3uk8qmd8kz4aupj7sjgyytiew9lwvoq4ksu3vkmeo63d86g31cqex8ye9psr29kl8oe3ca5fupya1/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://hftrcmjipxxw8yde542w5dmp2do0ngj2onjspxdbvq4gb73o8fgv1uq3e1yzugdysjv67p04a6zo4t0rsfxb55em81b6gyso9iajjr1h2xnghw7z6r2ri93ie44kn6b0:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 258}, "eval/total_success_rate": 0.699999988079071, "eval/total_success_rate_lb_0.01": 0.38132041692733765, "eval/total_success_rate_ub_0.01": 0.9409093260765076, "eval/total_success_rate_lb_0.025": 0.42515620589256287, "eval/total_success_rate_ub_0.025": 0.9237294793128967, "eval/total_success_rate_lb_0.05": 0.43590328097343445, "eval/total_success_rate_ub_0.05": 0.8963950872421265, "eval/total_success_rate_lb_0.1": 0.48208552598953247, "eval/total_success_rate_ub_0.1": 0.8360365033149719} \ No newline at end of file diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug-internal.log b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5d2ba0eaeca8c4c36ad87c17ac52ad55fa42e78e --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug-internal.log @@ -0,0 +1,758 @@ +2025-01-21 21:50:32,700 INFO StreamThr :2657635 [internal.py:wandb_internal():85] W&B internal server running at pid: 2657635, started at: 2025-01-21 21:50:32.700411 +2025-01-21 21:50:32,701 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status +2025-01-21 21:50:32,713 INFO WriterThread:2657635 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/run-q5zzaq6a.wandb +2025-01-21 21:50:32,713 DEBUG SenderThread:2657635 [sender.py:send():379] send: header +2025-01-21 21:50:32,722 DEBUG SenderThread:2657635 [sender.py:send():379] send: run +2025-01-21 21:50:33,001 INFO SenderThread:2657635 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files +2025-01-21 21:50:33,001 INFO SenderThread:2657635 [sender.py:_start_run_threads():1188] run started: q5zzaq6a with start time 1737514232.701994 +2025-01-21 21:50:33,014 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: check_version +2025-01-21 21:50:33,015 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: check_version +2025-01-21 21:50:33,085 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: run_start +2025-01-21 21:50:33,090 DEBUG HandlerThread:2657635 [system_info.py:__init__():26] System info init +2025-01-21 21:50:33,090 DEBUG HandlerThread:2657635 [system_info.py:__init__():41] System info init done +2025-01-21 21:50:33,090 INFO HandlerThread:2657635 [system_monitor.py:start():194] Starting system monitor +2025-01-21 21:50:33,090 INFO SystemMonitor:2657635 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-21 21:50:33,090 INFO HandlerThread:2657635 [system_monitor.py:probe():214] Collecting system info +2025-01-21 21:50:33,091 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started cpu monitoring +2025-01-21 21:50:33,091 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started disk monitoring +2025-01-21 21:50:33,092 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started gpu monitoring +2025-01-21 21:50:33,093 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started memory monitoring +2025-01-21 21:50:33,094 INFO SystemMonitor:2657635 [interfaces.py:start():188] Started network monitoring +2025-01-21 21:50:33,104 DEBUG HandlerThread:2657635 [system_info.py:probe():152] Probing system +2025-01-21 21:50:33,106 DEBUG HandlerThread:2657635 [system_info.py:_probe_git():137] Probing git +2025-01-21 21:50:33,110 DEBUG HandlerThread:2657635 [system_info.py:_probe_git():145] Probing git done +2025-01-21 21:50:33,110 DEBUG HandlerThread:2657635 [system_info.py:probe():200] Probing system done +2025-01-21 21:50:33,111 DEBUG HandlerThread:2657635 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-22T02:50:33.104693', 'startedAt': '2025-01-22T02:50:32.686458', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'dc9da9fa0ef2996c7fec8d105e7933e728cd952f'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.467375, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.963, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.915, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 418.30762100219727}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-21 21:50:33,111 INFO HandlerThread:2657635 [system_monitor.py:probe():224] Finished collecting system info +2025-01-21 21:50:33,111 INFO HandlerThread:2657635 [system_monitor.py:probe():227] Publishing system info +2025-01-21 21:50:33,111 DEBUG HandlerThread:2657635 [system_info.py:_save_code():44] Saving code +2025-01-21 21:50:33,112 DEBUG HandlerThread:2657635 [system_info.py:_save_code():67] Saving code done +2025-01-21 21:50:33,112 DEBUG HandlerThread:2657635 [system_info.py:_save_patches():84] Saving git patches +2025-01-21 21:50:33,174 DEBUG HandlerThread:2657635 [system_info.py:_save_patches():126] Saving git patches done +2025-01-21 21:50:33,174 INFO HandlerThread:2657635 [system_monitor.py:probe():229] Finished publishing system info +2025-01-21 21:50:33,177 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:50:33,177 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-21 21:50:33,178 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-21 21:50:33,178 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-21 21:50:33,410 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: python_packages +2025-01-21 21:50:33,410 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: python_packages +2025-01-21 21:50:33,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:50:33,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:33,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:50:33,423 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/80eig8ty-wandb-metadata.json +2025-01-21 21:50:33,457 INFO HandlerThread:2657635 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/tb" + save: true +} +control { + relay_id: "c13fbe6decbf490f88545b861eb62ce0" +} +_info { + stream_id: "q5zzaq6a" +} + +2025-01-21 21:50:33,457 DEBUG HandlerThread:2657635 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-21 21:50:33,467 INFO wandb-upload_2:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/83g0v4to-diff.patch +2025-01-21 21:50:33,472 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/cyxgueju-code/FISH/eval_robot.py +2025-01-21 21:50:33,562 DEBUG SenderThread:2657635 [sender.py:send():379] send: telemetry +2025-01-21 21:50:33,562 DEBUG SenderThread:2657635 [sender.py:send():379] send: tbrecord +2025-01-21 21:50:33,563 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:50:33,563 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 with policy live +2025-01-21 21:50:33,748 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/m119crax-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:50:34,003 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:50:34,003 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/diff.patch +2025-01-21 21:50:34,003 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/requirements.txt +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-metadata.json +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code/FISH/eval_robot.py +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code +2025-01-21 21:50:34,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code/FISH +2025-01-21 21:50:34,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:35,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:36,002 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:36,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:36,856 DEBUG SenderThread:2657635 [sender.py:send():379] send: config +2025-01-21 21:50:37,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:37,870 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:38,003 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:38,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:39,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:40,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:40,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:41,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:42,004 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:42,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:43,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:43,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:44,005 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:44,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:45,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:46,006 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:46,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:47,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:48,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:50:48,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:50:48,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:48,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:49,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:50,007 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:50,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:51,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:52,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:53,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:54,009 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:50:54,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:54,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:55,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:56,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:57,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:58,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:50:59,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:50:59,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:00,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:01,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:02,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:03,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:51:03,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:51:03,457 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:04,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:04,498 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:05,012 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml +2025-01-21 21:51:05,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:06,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:07,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:08,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:09,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:10,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:10,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:11,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:12,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:13,416 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:14,417 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:15,412 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:15,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:16,417 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:17,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:18,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:51:18,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:51:18,453 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:19,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:20,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:20,563 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:21,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:22,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:23,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:24,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:25,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:26,197 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:26,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:27,130 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:51:27,130 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 with policy now +2025-01-21 21:51:27,131 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:51:27,132 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:51:27,132 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:51:27,133 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:51:27,405 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/4ha7vfr4-media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 +2025-01-21 21:51:27,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:28,022 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:28,022 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 +2025-01-21 21:51:28,022 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:51:28,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:51:28,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos +2025-01-21 21:51:28,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media +2025-01-21 21:51:28,350 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/t3wrlr1c-media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 +2025-01-21 21:51:28,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:29,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:30,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:30,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:31,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:31,793 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:32,023 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:32,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:33,094 DEBUG SystemMonitor:2657635 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-21 21:51:33,095 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:51:33,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:51:33,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:51:33,453 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:34,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:35,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:36,025 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:36,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:37,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:37,532 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:38,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:39,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:40,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:41,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:42,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:42,583 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:43,027 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:51:43,294 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/8dp9bw6f-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:51:43,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:44,027 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:44,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:45,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:46,028 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:46,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:47,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:48,029 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:51:48,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:51:48,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:51:48,457 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:48,568 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:49,422 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:50,422 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:51,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:52,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:53,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:54,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:54,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:55,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:56,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:57,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:58,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:51:59,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:51:59,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:00,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:01,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:02,424 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:03,104 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:52:03,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:52:03,412 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:52:03,453 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:04,425 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:04,547 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:05,461 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:06,425 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:06,747 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:52:06,747 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 with policy now +2025-01-21 21:52:06,747 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:52:06,748 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:52:06,748 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:52:06,748 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:52:06,959 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/k6evo0up-media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 +2025-01-21 21:52:07,035 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:52:07,035 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 +2025-01-21 21:52:07,035 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:52:07,338 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/04wk10ls-media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 +2025-01-21 21:52:07,426 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:08,035 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:08,427 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:09,036 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:09,433 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:10,165 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:10,430 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:11,430 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:12,037 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:12,430 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:13,428 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:14,038 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:14,428 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:14,848 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:52:14,849 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:52:14,849 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:52:15,038 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:52:15,038 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:52:15,428 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:15,549 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:16,038 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:16,429 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:17,428 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:18,039 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:18,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:52:18,411 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:52:18,440 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:19,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:20,040 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:20,440 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:20,557 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:21,441 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:22,441 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:23,441 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:24,441 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:25,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:25,877 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:26,043 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:26,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:27,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:28,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:29,442 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:30,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:31,416 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:31,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:32,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:33,095 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:52:33,411 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:52:33,412 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:52:33,453 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:34,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:35,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:36,443 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:36,525 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:37,449 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:38,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:38,255 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:52:38,255 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 with policy now +2025-01-21 21:52:38,257 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:52:38,258 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:52:38,258 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:52:38,259 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:52:38,445 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:38,485 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/81r70su4-media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 +2025-01-21 21:52:39,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:52:39,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:39,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 +2025-01-21 21:52:39,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:52:39,335 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/fuaftwst-media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 +2025-01-21 21:52:39,451 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:40,047 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:40,450 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:41,048 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:41,456 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:41,686 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:42,460 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:43,455 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:44,049 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:44,461 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:45,455 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:46,049 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:46,459 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:47,461 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:47,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:48,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:52:48,414 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:52:48,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:49,051 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:52:49,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:49,672 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:52:49,672 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:52:49,690 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:52:49,810 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/kxiyxen9-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:52:50,051 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:52:50,051 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:50,468 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:51,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:52,052 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:52,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:52,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:53,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:54,052 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:52:54,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:55,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:56,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:57,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:58,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:52:58,462 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:52:59,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:00,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:01,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:02,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:03,096 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:53:03,413 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:53:03,414 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:53:03,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:03,573 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:04,463 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:05,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:06,465 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:07,464 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:08,057 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:08,465 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:09,418 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:09,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:10,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:11,465 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:12,465 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:13,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:14,419 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:14,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:15,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:16,466 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:17,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:18,414 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:53:18,414 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:53:18,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:19,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:19,529 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:20,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:21,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:22,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:23,467 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:24,468 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:25,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:25,468 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:26,468 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:27,469 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:28,066 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:28,214 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:53:28,214 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 with policy now +2025-01-21 21:53:28,218 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:53:28,218 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:53:28,218 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:53:28,233 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:53:28,463 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/pa105z2x-media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 +2025-01-21 21:53:28,469 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:29,064 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:53:29,064 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 +2025-01-21 21:53:29,064 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:53:29,366 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/ykc075sz-media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 +2025-01-21 21:53:29,477 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:30,064 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:30,420 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:30,474 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:31,477 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:32,479 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:33,096 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:53:33,416 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:53:33,417 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:53:33,475 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:34,066 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:34,475 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:35,475 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:35,689 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:36,066 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:36,478 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:37,477 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:38,479 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:39,482 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:39,732 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:53:39,732 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:53:39,733 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:53:40,068 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:53:40,068 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:53:40,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:41,480 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:41,661 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:42,069 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:42,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:43,482 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:44,480 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:45,480 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:46,070 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:46,480 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:46,812 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:47,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:48,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:53:48,415 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:53:48,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:49,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:50,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:51,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:52,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:52,625 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:53,481 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:54,073 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:53:54,482 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:55,483 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:56,483 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:57,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:58,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:53:58,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:53:59,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:00,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:01,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:02,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:03,097 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:54:03,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:54:03,415 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:54:03,484 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:03,501 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:04,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:05,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:06,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:07,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:08,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:09,423 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:09,485 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:10,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:11,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:12,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:13,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:14,425 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:14,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:15,486 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:16,487 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:17,487 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:18,415 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:54:18,415 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:54:18,487 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:19,426 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:19,469 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:54:19,469 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 with policy now +2025-01-21 21:54:19,474 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:54:19,475 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:19,475 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:19,475 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:19,489 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:19,752 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/n2bj6gp6-media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 +2025-01-21 21:54:20,082 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:20,082 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:20,082 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 +2025-01-21 21:54:20,082 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval +2025-01-21 21:54:20,389 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/ayptpw1r-media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 +2025-01-21 21:54:20,490 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:21,497 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:22,083 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:22,499 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:23,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:24,083 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:24,431 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:24,506 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:25,500 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:26,501 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:27,499 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:28,085 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:28,515 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:29,436 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:29,512 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:30,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:31,506 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:32,507 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:33,103 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:54:33,422 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:54:33,422 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:54:33,506 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:34,454 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:34,501 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:34,799 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:34,799 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:34,800 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:35,087 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:35,087 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:35,292 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:54:35,293 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:54:35,346 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/dc1eu72r-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:35,476 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:54:35,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:35,826 INFO wandb-upload_0:2657635 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp72xx187r +2025-01-21 21:54:35,862 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-21 21:54:35,870 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: partial_history +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table +2025-01-21 21:54:36,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval +2025-01-21 21:54:36,245 INFO SenderThread:2657635 [sender.py:send_request_log_artifact():1525] logged artifact run-q5zzaq6a-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcwMTgwMg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMTQxMA==', 'latestArtifact': None}} +2025-01-21 21:54:36,245 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:54:36,245 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json with policy now +2025-01-21 21:54:36,245 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:36,245 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,246 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,246 DEBUG SenderThread:2657635 [sender.py:send():379] send: telemetry +2025-01-21 21:54:36,246 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,250 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,250 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,250 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,250 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,251 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,251 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,251 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,251 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,252 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,252 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,252 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,252 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,253 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,253 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,253 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,253 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,254 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,254 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,255 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,255 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:36,255 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:36,255 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: log_artifact +2025-01-21 21:54:36,436 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/104pge0x-media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json +2025-01-21 21:54:36,466 INFO wandb-upload_2:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/rkbjsa9i-media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json +2025-01-21 21:54:36,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:36,680 INFO wandb-upload_0:2657635 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpivxe_ti9 +2025-01-21 21:54:36,739 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/0ukvjqy0-media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json +2025-01-21 21:54:37,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:37,116 INFO SenderThread:2657635 [sender.py:send_request_log_artifact():1525] logged artifact run-q5zzaq6a-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQ1NzcwMTgyMA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjU0NTIwMTQxMQ==', 'latestArtifact': None}} +2025-01-21 21:54:37,117 DEBUG SenderThread:2657635 [sender.py:send():379] send: files +2025-01-21 21:54:37,117 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json with policy now +2025-01-21 21:54:37,117 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:37,117 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:37,118 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:37,389 INFO wandb-upload_2:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/j03x1hmm-media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json +2025-01-21 21:54:37,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:38,088 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:38,089 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:38,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:39,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:40,431 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:40,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:41,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:42,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:43,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:44,502 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:45,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:46,431 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:46,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:47,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:48,416 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: stop_status +2025-01-21 21:54:48,416 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: stop_status +2025-01-21 21:54:48,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:49,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:50,503 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:51,472 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:51,613 DEBUG SenderThread:2657635 [sender.py:send():379] send: exit +2025-01-21 21:54:51,613 INFO SenderThread:2657635 [sender.py:send_exit():586] handling exit code: 0 +2025-01-21 21:54:51,613 INFO SenderThread:2657635 [sender.py:send_exit():588] handling runtime: 258 +2025-01-21 21:54:51,613 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:send_exit():594] send defer +2025-01-21 21:54:51,614 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,614 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-21 21:54:51,614 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 1 +2025-01-21 21:54:51,614 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,614 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-21 21:54:51,614 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-21 21:54:51,614 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 2 +2025-01-21 21:54:51,615 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,615 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-21 21:54:51,615 INFO HandlerThread:2657635 [system_monitor.py:finish():203] Stopping system monitor +2025-01-21 21:54:51,615 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined cpu monitor +2025-01-21 21:54:51,615 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined disk monitor +2025-01-21 21:54:51,615 DEBUG SystemMonitor:2657635 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-21 21:54:51,616 DEBUG SystemMonitor:2657635 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-21 21:54:51,667 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined gpu monitor +2025-01-21 21:54:51,667 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined memory monitor +2025-01-21 21:54:51,667 INFO HandlerThread:2657635 [interfaces.py:finish():200] Joined network monitor +2025-01-21 21:54:51,668 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:51,668 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-21 21:54:51,668 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 3 +2025-01-21 21:54:51,668 DEBUG SenderThread:2657635 [sender.py:send():379] send: stats +2025-01-21 21:54:51,668 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,668 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-21 21:54:51,668 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:51,669 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-21 21:54:51,669 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 4 +2025-01-21 21:54:51,669 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:51,669 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-21 21:54:52,094 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml +2025-01-21 21:54:52,094 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:54,095 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:57,804 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:57,804 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:57,804 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: keepalive +2025-01-21 21:54:57,805 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:57,805 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-21 21:54:57,805 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 5 +2025-01-21 21:54:57,805 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:57,806 DEBUG SenderThread:2657635 [sender.py:send():379] send: history +2025-01-21 21:54:57,806 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: summary_record +2025-01-21 21:54:57,806 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:57,806 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-21 21:54:57,806 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:57,807 DEBUG SenderThread:2657635 [sender.py:send():379] send: summary +2025-01-21 21:54:57,808 INFO SenderThread:2657635 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-21 21:54:57,808 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:57,808 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-21 21:54:57,809 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 6 +2025-01-21 21:54:57,809 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:57,809 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-21 21:54:57,809 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:57,809 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-21 21:54:57,809 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 7 +2025-01-21 21:54:57,810 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: status_report +2025-01-21 21:54:57,810 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:57,810 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-21 21:54:57,810 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:57,810 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-21 21:54:58,096 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:58,471 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:58,855 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 8 +2025-01-21 21:54:58,855 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:58,856 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:58,856 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-21 21:54:58,856 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:58,856 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-21 21:54:58,856 INFO SenderThread:2657635 [job_builder.py:build():440] Attempting to build job artifact +2025-01-21 21:54:58,859 INFO SenderThread:2657635 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-21 21:54:58,860 INFO SenderThread:2657635 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-21 21:54:58,870 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 9 +2025-01-21 21:54:58,870 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:58,870 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-21 21:54:58,871 DEBUG SenderThread:2657635 [sender.py:send():379] send: artifact +2025-01-21 21:54:59,096 INFO SenderThread:2657635 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQ1NzY3OTQ4Mg==', 'versionIndex': 456}}} +2025-01-21 21:54:59,096 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,096 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-21 21:54:59,096 INFO SenderThread:2657635 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-21 21:54:59,097 INFO Thread-12 :2657635 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:59,097 INFO SenderThread:2657635 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files +2025-01-21 21:54:59,097 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml config.yaml +2025-01-21 21:54:59,097 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json wandb-summary.json +2025-01-21 21:54:59,098 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/requirements.txt requirements.txt +2025-01-21 21:54:59,098 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-metadata.json wandb-metadata.json +2025-01-21 21:54:59,098 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:59,098 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log output.log +2025-01-21 21:54:59,100 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/diff.patch diff.patch +2025-01-21 21:54:59,103 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 media/videos/eval/4_eval_4_a207dd3be5225e41ea05.mp4 +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 media/videos/eval/3_eval_3_ba9216e7dc0717aa3197.mp4 +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 media/videos/eval/1_eval_1_4ec83cc214455bb6907b.mp4 +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 media/videos/eval/2_eval_2_e7f6fc91409ff331f77f.mp4 +2025-01-21 21:54:59,105 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 media/videos/eval/0_eval_0_7fe7dc19834ce97a254b.mp4 +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json media/table/eval/total_success_rate_ci_6_36a4b75cedb5a99e4ccb.table.json +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json media/table/eval/success_rate_ci_5_c77850ebfe62f375339b.table.json +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 10 +2025-01-21 21:54:59,107 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,107 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-21 21:54:59,107 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-21 21:54:59,107 INFO SenderThread:2657635 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:54:59,295 INFO wandb-upload_0:2657635 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/wandb-summary.json +2025-01-21 21:54:59,299 INFO wandb-upload_2:2657635 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/requirements.txt +2025-01-21 21:54:59,347 INFO wandb-upload_1:2657635 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/config.yaml +2025-01-21 21:54:59,353 INFO wandb-upload_4:2657635 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/files/output.log +2025-01-21 21:54:59,418 INFO wandb-upload_3:2657635 [upload_job.py:push():130] Uploaded file /tmp/tmpv50oya3jwandb/691vlpdl-events.out.tfevents.1737514233.leonmkim-ROG-Strix-G15CS-G15CS.2657441.0 +2025-01-21 21:54:59,471 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:59,471 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:59,619 INFO Thread-11 (_thread_body):2657635 [sender.py:transition_state():614] send defer: 11 +2025-01-21 21:54:59,619 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,619 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-21 21:54:59,619 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,619 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-21 21:54:59,619 INFO SenderThread:2657635 [file_pusher.py:join():175] waiting for file pusher +2025-01-21 21:54:59,620 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 12 +2025-01-21 21:54:59,620 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,620 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-21 21:54:59,621 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,621 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-21 21:54:59,621 INFO SenderThread:2657635 [file_stream.py:finish():601] file stream finish called +2025-01-21 21:54:59,697 INFO SenderThread:2657635 [file_stream.py:finish():605] file stream finish is done +2025-01-21 21:54:59,697 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 13 +2025-01-21 21:54:59,698 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,698 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-21 21:54:59,698 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,698 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-21 21:54:59,698 INFO SenderThread:2657635 [sender.py:transition_state():614] send defer: 14 +2025-01-21 21:54:59,698 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: defer +2025-01-21 21:54:59,698 INFO HandlerThread:2657635 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-21 21:54:59,698 DEBUG SenderThread:2657635 [sender.py:send():379] send: final +2025-01-21 21:54:59,699 DEBUG SenderThread:2657635 [sender.py:send():379] send: footer +2025-01-21 21:54:59,699 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: defer +2025-01-21 21:54:59,699 INFO SenderThread:2657635 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-21 21:54:59,699 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:59,699 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:59,700 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: server_info +2025-01-21 21:54:59,700 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-21 21:54:59,700 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: server_info +2025-01-21 21:54:59,700 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: get_summary +2025-01-21 21:54:59,702 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-21 21:54:59,703 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-21 21:54:59,758 DEBUG SenderThread:2657635 [sender.py:send_request():406] send_request: poll_exit +2025-01-21 21:54:59,758 INFO MainThread:2657635 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-21 21:54:59,759 INFO MainThread:2657635 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-21 21:54:59,759 INFO MainThread:2657635 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-21 21:54:59,759 DEBUG HandlerThread:2657635 [handler.py:handle_request():158] handle_request: shutdown +2025-01-21 21:54:59,759 INFO HandlerThread:2657635 [handler.py:finish():882] shutting down handler +2025-01-21 21:55:00,700 INFO WriterThread:2657635 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/run-q5zzaq6a.wandb +2025-01-21 21:55:00,758 INFO SenderThread:2657635 [sender.py:finish():1615] shutting down sender +2025-01-21 21:55:00,758 INFO SenderThread:2657635 [file_pusher.py:finish():169] shutting down file pusher +2025-01-21 21:55:00,759 INFO SenderThread:2657635 [file_pusher.py:join():175] waiting for file pusher diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug.log b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..f026f0f970acfff54624956d75308a4a4c218fb1 --- /dev/null +++ b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Configure stats pid to 2657441 +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/settings +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-21 21:50:32,691 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug.log +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/logs/debug-internal.log +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:init():569] calling init triggers +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 2, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '4295_2', 'wandb_notes': '4295_2_restarted_1', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': False, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 7, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'restore_missing_checkpoint': None, 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 2, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'crop_distractors_tlhw': [32, 68, 174, 174], 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False, 'crop_distractors_tlhw': [32, 68, 174, 174]}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'test': {'username': 'leonmkim', 'hostname': 'grasp-login1', 'name_of_expert_demo': '112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'root_dir': '~/fish_leon', 'expert_dataset_dirpath': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '~/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'batch_size': 128, 'store_dataset_in_memory': False, 'use_tb': True, 'local_snapshot_root_dir': '/mnt/bighdd/fish_contact_backup', 'local_snapshot_dir': '/mnt/bighdd/fish_contact_backup/exp_local/frankagym_pixels/FrankaInsertion-v1', 'resume_wandb_run': False}, 'feature_type': '180x240_crpdstlhw24x51x130x130_1_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthist_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 7500, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a'} +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:init():619] starting backend +2025-01-21 21:50:32,692 INFO MainThread:2657441 [wandb_init.py:init():623] setting up manager +2025-01-21 21:50:32,700 INFO MainThread:2657441 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-21 21:50:32,701 INFO MainThread:2657441 [wandb_init.py:init():631] backend started and connected +2025-01-21 21:50:32,712 INFO MainThread:2657441 [wandb_init.py:init():720] updated telemetry +2025-01-21 21:50:32,719 INFO MainThread:2657441 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-21 21:50:33,014 INFO MainThread:2657441 [wandb_run.py:_on_init():2435] communicating current version +2025-01-21 21:50:33,080 INFO MainThread:2657441 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.4 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-21 21:50:33,080 INFO MainThread:2657441 [wandb_init.py:init():804] starting run threads in backend +2025-01-21 21:50:33,411 INFO MainThread:2657441 [wandb_run.py:_console_start():2413] atexit reg +2025-01-21 21:50:33,411 INFO MainThread:2657441 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-21 21:50:33,411 INFO MainThread:2657441 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-21 21:50:33,411 INFO MainThread:2657441 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-21 21:50:33,412 INFO MainThread:2657441 [wandb_init.py:init():847] run started, returning control to user process +2025-01-21 21:50:33,412 INFO MainThread:2657441 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/q5zzaq6a/tb, True +2025-01-21 21:50:36,855 INFO MainThread:2657441 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-21 21:55:00,760 WARNING MsgRouterThr:2657441 [router.py:message_loop():77] message_loop has been closed diff --git a/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/run-q5zzaq6a.wandb b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/run-q5zzaq6a.wandb new file mode 100644 index 0000000000000000000000000000000000000000..97c21544d3ee607160029bdceed05502c9dff60b Binary files /dev/null and b/q5zzaq6a/wandb/run-20250121_215032-q5zzaq6a/run-q5zzaq6a.wandb differ diff --git a/snapshot_10999.pt b/snapshot_10999.pt new file mode 100644 index 0000000000000000000000000000000000000000..b01fcd419d47d4972d5395ba48abff418517823d --- /dev/null +++ b/snapshot_10999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df6581b067b8cb4d686fe6035ec563f17a122379129e2428c0fb94adf042c7e0 +size 910332934 diff --git a/snapshot_11749.pt b/snapshot_11749.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6b61d00810e9dd2201cffff5eb16d838cfaa443 --- /dev/null +++ b/snapshot_11749.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f190c177970ee28916b860db5af9a64e36066b20cd3f3d047fa8b64404c682b2 +size 910332934 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..3f40b08da119d72c0893e836e126a63212e65482 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b203a953e17485d9b55a7b675d8c4c482045b9a6f729cefde628bc38b232ffd +size 910332934 diff --git a/snapshot_13500.pt b/snapshot_13500.pt new file mode 100644 index 0000000000000000000000000000000000000000..1152ec558ca47d624db25ded7880c39d71598b3d --- /dev/null +++ b/snapshot_13500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9d47a7381cdf3cd002091d069188469a1f36df48b5edfed4083619d80ac3ec2e +size 910332934 diff --git a/snapshot_15000.pt b/snapshot_15000.pt new file mode 100644 index 0000000000000000000000000000000000000000..194dc345ed0fcf081ee1b5dd0b178cfe2ac6551d --- /dev/null +++ b/snapshot_15000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a64babd57a5f2c3de66eb79006f2922cf6c9d7a5c4999cdffc3942438d3a892 +size 910332934 diff --git a/snapshot_7499.pt b/snapshot_7499.pt new file mode 100644 index 0000000000000000000000000000000000000000..d869b02c510d2a1235dd09a00830c67804a8e843 --- /dev/null +++ b/snapshot_7499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:baf59ba0dcfbca500f31acf0bc5f22034d6f34edd6b1128fe45155c31da4b797 +size 250961920 diff --git a/snapshot_last.pt b/snapshot_last.pt new file mode 100644 index 0000000000000000000000000000000000000000..194dc345ed0fcf081ee1b5dd0b178cfe2ac6551d --- /dev/null +++ b/snapshot_last.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a64babd57a5f2c3de66eb79006f2922cf6c9d7a5c4999cdffc3942438d3a892 +size 910332934 diff --git a/vaakmemz/.hydra/config.yaml b/vaakmemz/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..836895e41a10194087484706499cf57e8cf16e48 --- /dev/null +++ b/vaakmemz/.hydra/config.yaml @@ -0,0 +1,352 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '4295_2' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 7500 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + crop_distractors_tlhw: null + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + crop_distractors_tlhw: ${agent.config.policy_cfg.crop_distractors_tlhw} +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/vaakmemz/.hydra/hydra.yaml b/vaakmemz/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e6005b7311e4944b7fe8e1d734d8f054d320271 --- /dev/null +++ b/vaakmemz/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/4295_2/vaakmemz + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/vaakmemz/.hydra/overrides.yaml b/vaakmemz/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/vaakmemz/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/vaakmemz/eval_policy.log b/vaakmemz/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..e4f18d0238859999a4bdf87753a992c15885b371 --- /dev/null +++ b/vaakmemz/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-21 21:30:33,740][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:429: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-21 21:30:33,743][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:366: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-21 21:30:34,856][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:415: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) +