diff --git a/.gitattributes b/.gitattributes index 0af49d89eb68d24804860cbb8c7bf4fa68396320..1c24fa411f027f33d17999f927770892165f7e93 100644 --- a/.gitattributes +++ b/.gitattributes @@ -63,3 +63,34 @@ e3yh61zm/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text 2uvisx3r/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text 2uvisx3r/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text 2uvisx3r/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +e3yh61zm/episode_rosbags/episode_1_2025-01-07-17-56-35.bag filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +i3pbiz9p/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/episode_rosbags/episode_3_2025-01-06-20-38-45.bag filter=lfs diff=lfs merge=lfs -text +e3yh61zm/episode_rosbags/episode_2_2025-01-07-17-57-28.bag filter=lfs diff=lfs merge=lfs -text +vd7nl959/episode_rosbags/episode_2_2025-01-06-20-38-01.bag filter=lfs diff=lfs merge=lfs -text +vd7nl959/episode_rosbags/episode_1_2025-01-06-20-37-08.bag filter=lfs diff=lfs merge=lfs -text +kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 filter=lfs diff=lfs merge=lfs -text +kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 filter=lfs diff=lfs merge=lfs -text +kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/0_eval_0_79f998250d76be901393.mp4 filter=lfs diff=lfs merge=lfs -text +kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 filter=lfs diff=lfs merge=lfs -text +kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 filter=lfs diff=lfs merge=lfs -text +kq00a7xd/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kq00a7xd/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kq00a7xd/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kq00a7xd/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +kq00a7xd/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +vd7nl959/episode_rosbags/episode_0_2025-01-06-20-36-28.bag filter=lfs diff=lfs merge=lfs -text +vd7nl959/episode_rosbags/episode_4_2025-01-06-20-39-25.bag filter=lfs diff=lfs merge=lfs -text +kq00a7xd/episode_rosbags/episode_1_2025-01-07-18-02-10.bag filter=lfs diff=lfs merge=lfs -text +kq00a7xd/episode_rosbags/episode_0_2025-01-07-18-01-28.bag filter=lfs diff=lfs merge=lfs -text +kq00a7xd/episode_rosbags/episode_4_2025-01-07-18-04-36.bag filter=lfs diff=lfs merge=lfs -text +2uvisx3r/episode_rosbags/episode_2_2025-01-07-20-36-41.bag filter=lfs diff=lfs merge=lfs -text diff --git a/2uvisx3r/episode_rosbags/episode_2_2025-01-07-20-36-41.bag b/2uvisx3r/episode_rosbags/episode_2_2025-01-07-20-36-41.bag new file mode 100644 index 0000000000000000000000000000000000000000..151c4683de51d50afc47cf5f356bea154ef938df --- /dev/null +++ b/2uvisx3r/episode_rosbags/episode_2_2025-01-07-20-36-41.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd22b4510e78461312cf7d8b1258027cca8654ec644579bbfa04116dff3cb1e2 +size 1402578446 diff --git a/917c47lf/episode_rosbags/aligned_depth_to_color_K.npy b/917c47lf/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/917c47lf/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/917c47lf/episode_rosbags/cam_tf_world.npy b/917c47lf/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/917c47lf/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/917c47lf/episode_rosbags/color_K.npy b/917c47lf/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/917c47lf/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/917c47lf/episode_rosbags/depth_K.npy b/917c47lf/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/917c47lf/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/917c47lf/eval_robot.log b/917c47lf/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..746f44f79e0c450357c8f2a296a68d5816075c6a --- /dev/null +++ b/917c47lf/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 22:18:03,986][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 22:18:03,988][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/917c47lf/wandb/debug-internal.log b/917c47lf/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..b322029fa0a3a4f25a70bfa63b54b4ba8a463004 --- /dev/null +++ b/917c47lf/wandb/debug-internal.log @@ -0,0 +1,767 @@ +2025-01-06 22:18:10,159 INFO StreamThr :2123564 [internal.py:wandb_internal():85] W&B internal server running at pid: 2123564, started at: 2025-01-06 22:18:10.159088 +2025-01-06 22:18:10,160 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status +2025-01-06 22:18:10,172 INFO WriterThread:2123564 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/run-917c47lf.wandb +2025-01-06 22:18:10,172 DEBUG SenderThread:2123564 [sender.py:send():379] send: header +2025-01-06 22:18:10,181 DEBUG SenderThread:2123564 [sender.py:send():379] send: run +2025-01-06 22:18:10,412 INFO SenderThread:2123564 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files +2025-01-06 22:18:10,413 INFO SenderThread:2123564 [sender.py:_start_run_threads():1188] run started: 917c47lf with start time 1736219890.1595 +2025-01-06 22:18:10,429 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 22:18:10,429 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: check_version +2025-01-06 22:18:10,499 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 22:18:10,504 DEBUG HandlerThread:2123564 [system_info.py:__init__():26] System info init +2025-01-06 22:18:10,504 DEBUG HandlerThread:2123564 [system_info.py:__init__():41] System info init done +2025-01-06 22:18:10,504 INFO HandlerThread:2123564 [system_monitor.py:start():194] Starting system monitor +2025-01-06 22:18:10,504 INFO SystemMonitor:2123564 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 22:18:10,504 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 22:18:10,504 INFO HandlerThread:2123564 [system_monitor.py:probe():214] Collecting system info +2025-01-06 22:18:10,505 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started disk monitoring +2025-01-06 22:18:10,506 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 22:18:10,507 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started memory monitoring +2025-01-06 22:18:10,507 INFO SystemMonitor:2123564 [interfaces.py:start():188] Started network monitoring +2025-01-06 22:18:10,518 DEBUG HandlerThread:2123564 [system_info.py:probe():152] Probing system +2025-01-06 22:18:10,520 DEBUG HandlerThread:2123564 [system_info.py:_probe_git():137] Probing git +2025-01-06 22:18:10,524 DEBUG HandlerThread:2123564 [system_info.py:_probe_git():145] Probing git done +2025-01-06 22:18:10,524 DEBUG HandlerThread:2123564 [system_info.py:probe():200] Probing system done +2025-01-06 22:18:10,524 DEBUG HandlerThread:2123564 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T03:18:10.518603', 'startedAt': '2025-01-07T03:18:10.150271', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3562.4985, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 4500.056, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 564.8245735168457}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 22:18:10,524 INFO HandlerThread:2123564 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 22:18:10,524 INFO HandlerThread:2123564 [system_monitor.py:probe():227] Publishing system info +2025-01-06 22:18:10,524 DEBUG HandlerThread:2123564 [system_info.py:_save_code():44] Saving code +2025-01-06 22:18:10,526 DEBUG HandlerThread:2123564 [system_info.py:_save_code():67] Saving code done +2025-01-06 22:18:10,526 DEBUG HandlerThread:2123564 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 22:18:10,588 DEBUG HandlerThread:2123564 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 22:18:10,589 INFO HandlerThread:2123564 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 22:18:10,592 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:18:10,592 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 22:18:10,592 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 22:18:10,592 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 22:18:10,822 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 22:18:10,823 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: python_packages +2025-01-06 22:18:10,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:18:10,824 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/6tpjjiq2-wandb-metadata.json +2025-01-06 22:18:10,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:10,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:18:10,826 INFO HandlerThread:2123564 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/tb" + save: true +} +control { + relay_id: "cef7d876c4ad4fd199c5eb1a9bd32925" +} +_info { + stream_id: "917c47lf" +} + +2025-01-06 22:18:10,826 DEBUG HandlerThread:2123564 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 22:18:10,851 INFO wandb-upload_2:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/kdr9sx7r-diff.patch +2025-01-06 22:18:10,855 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/5sqz2j7o-code/FISH/eval_robot.py +2025-01-06 22:18:10,976 DEBUG SenderThread:2123564 [sender.py:send():379] send: telemetry +2025-01-06 22:18:10,976 DEBUG SenderThread:2123564 [sender.py:send():379] send: tbrecord +2025-01-06 22:18:10,977 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:18:10,977 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 with policy live +2025-01-06 22:18:11,156 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/v1x7seua-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:18:11,414 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/requirements.txt +2025-01-06 22:18:11,414 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:18:11,414 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/diff.patch +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-metadata.json +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/code/FISH/eval_robot.py +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/code +2025-01-06 22:18:11,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/code/FISH +2025-01-06 22:18:11,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:12,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:13,413 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:13,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:14,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:15,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:15,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:16,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:17,522 DEBUG SenderThread:2123564 [sender.py:send():379] send: config +2025-01-06 22:18:17,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:18,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:19,415 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:19,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:20,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:20,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:21,416 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:21,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:22,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:23,416 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:23,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:24,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:25,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:18:25,823 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:18:25,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:25,972 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:26,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:27,417 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:27,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:28,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:29,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:30,825 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:30,979 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:31,418 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:31,826 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:32,826 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:33,419 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:33,826 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:34,826 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:35,420 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:18:35,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:36,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:36,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:37,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:38,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:39,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:40,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:18:40,823 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:18:40,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:41,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:41,930 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:42,422 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml +2025-01-06 22:18:42,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:43,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:44,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:45,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:46,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:47,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:47,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:48,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:49,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:50,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:51,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:52,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:52,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:53,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:54,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:55,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:18:55,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:18:55,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:56,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:57,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:57,962 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:18:58,829 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:18:59,829 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:00,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:01,662 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:19:01,662 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 with policy now +2025-01-06 22:19:01,662 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:19:01,663 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:19:01,663 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:19:01,664 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:19:01,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:01,901 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/37axuafr-media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:19:02,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media +2025-01-06 22:19:02,683 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/uc6srzzt-media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 +2025-01-06 22:19:02,837 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:03,427 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:03,674 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:03,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:04,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:05,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:06,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:07,429 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:07,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:08,833 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:08,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:09,429 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:09,838 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:10,507 DEBUG SystemMonitor:2123564 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 22:19:10,508 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:19:10,823 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:19:10,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:19:10,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:11,430 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:19:11,666 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/mzeyzwh7-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:19:11,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:12,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:13,431 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:13,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:14,346 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:14,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:15,432 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:15,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:16,839 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:17,432 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:17,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:18,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:19,829 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:19,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:20,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:21,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:22,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:23,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:24,840 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:25,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:19:25,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:19:25,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:25,905 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:26,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:27,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:28,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:29,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:30,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:31,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:31,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:32,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:33,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:34,842 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:35,438 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:35,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:36,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:36,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:37,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:38,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:39,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:40,508 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:19:40,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:19:40,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:19:40,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:41,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:42,088 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:42,576 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:19:42,577 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:19:42,577 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 with policy now +2025-01-06 22:19:42,577 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:19:42,577 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:19:42,578 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:19:42,791 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/qpwc9feh-media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 +2025-01-06 22:19:42,845 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:43,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:43,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:19:43,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 +2025-01-06 22:19:43,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:19:43,713 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/tnputsod-media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 +2025-01-06 22:19:43,847 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:44,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:45,441 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:45,845 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:46,845 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:47,442 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:47,830 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:47,845 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:48,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:49,443 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:49,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:50,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:51,225 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:19:51,226 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:19:51,226 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:19:51,443 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:51,444 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:19:51,444 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:19:51,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:52,851 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:53,444 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:53,637 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:53,852 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:54,852 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:55,445 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:55,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:19:55,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:19:55,847 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:56,848 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:57,446 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:57,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:58,789 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:19:58,861 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:19:59,446 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:19:59,863 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:00,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:01,863 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:02,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:03,830 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:03,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:04,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:05,862 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:06,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:07,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:08,837 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:08,864 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:09,864 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:10,509 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:20:10,824 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:20:10,824 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:20:10,869 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:11,864 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:12,867 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:13,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:13,974 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:14,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:15,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:16,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:17,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:18,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:19,834 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:19,866 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:20,868 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:21,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:22,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:23,865 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:24,868 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:24,882 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:25,180 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:20:25,180 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 with policy now +2025-01-06 22:20:25,182 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:20:25,183 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:20:25,183 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:20:25,183 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:20:25,397 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/xkkmrbaq-media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 +2025-01-06 22:20:25,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:25,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:20:25,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 +2025-01-06 22:20:25,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:20:25,728 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/u12h7cjr-media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 +2025-01-06 22:20:25,829 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:20:25,829 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:20:25,873 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:26,876 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:27,455 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:27,874 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:28,879 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:29,456 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:29,877 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:30,365 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:30,877 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:31,882 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:32,885 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:33,457 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:33,887 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:34,890 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:35,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:35,888 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:36,889 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:37,895 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:38,898 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:39,904 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:40,509 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:20:40,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:20:40,829 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:20:40,899 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:40,969 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:41,897 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:42,903 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:43,910 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:44,905 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:45,905 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:46,843 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:46,910 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:47,911 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:48,909 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:49,905 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:50,914 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:51,852 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:51,912 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:52,909 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:53,464 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:20:53,706 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/b8z1ithe-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:20:53,909 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:54,075 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:20:54,075 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:20:54,076 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:20:54,464 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:20:54,914 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:55,464 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:55,835 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:20:55,835 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:20:55,912 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:56,913 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:56,926 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:20:57,915 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:58,913 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:20:59,466 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:20:59,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:00,917 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:01,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:02,835 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:02,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:03,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:04,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:05,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:06,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:07,835 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:07,918 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:08,919 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:09,919 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:10,510 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:21:10,827 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:21:10,827 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:21:10,919 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:11,919 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:12,886 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:12,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:13,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:14,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:15,471 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:15,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:16,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:17,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:18,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:18,921 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:19,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:20,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:21,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:22,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:23,836 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:23,922 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:24,924 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:25,474 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:25,608 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:21:25,608 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 with policy now +2025-01-06 22:21:25,613 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:21:25,614 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:21:25,614 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:21:25,617 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:21:25,816 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/5k2hh8oi-media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 +2025-01-06 22:21:25,835 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:21:25,835 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:21:25,929 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:26,475 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:21:26,475 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 +2025-01-06 22:21:26,475 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:21:26,728 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/jpkayfdj-media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 +2025-01-06 22:21:26,935 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:27,475 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:27,930 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:28,928 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:28,933 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:29,932 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:30,936 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:31,477 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:31,936 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:32,935 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:33,477 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:33,938 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:34,121 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:34,937 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:35,941 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:36,934 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:37,943 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:38,946 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:39,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:39,949 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:40,511 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:21:40,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:21:40,844 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:21:40,946 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:41,948 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:42,135 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:21:42,135 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:21:42,135 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:21:42,481 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:21:42,481 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:21:42,948 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:43,481 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:43,971 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:44,966 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:45,384 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:45,952 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:46,952 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:47,482 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:47,952 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:48,956 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:49,955 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:50,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:50,953 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:51,954 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:52,953 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:53,953 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:54,961 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:55,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:21:55,828 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:21:55,962 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:55,979 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:21:56,962 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:57,486 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:21:57,965 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:58,962 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:21:59,963 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:00,963 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:01,844 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:01,963 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:02,964 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:03,964 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:04,964 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:05,968 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:06,846 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:06,966 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:07,966 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:08,965 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:09,965 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:10,511 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:22:10,828 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:22:10,828 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:22:10,967 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:11,490 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:11,880 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:11,967 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:12,968 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:13,976 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:14,598 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:22:14,598 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 with policy now +2025-01-06 22:22:14,607 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:22:14,608 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:14,608 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:14,609 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:14,811 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/ky50jorc-media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 +2025-01-06 22:22:14,982 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:15,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:15,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:15,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 +2025-01-06 22:22:15,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval +2025-01-06 22:22:15,758 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/ngi6k4n8-media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 +2025-01-06 22:22:15,986 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:16,986 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:17,492 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:17,850 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:17,982 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:19,006 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:19,493 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:19,987 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:20,994 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:21,494 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:21,999 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:22,851 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:22,997 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:23,495 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:24,006 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:25,009 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:25,841 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 22:22:25,841 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: stop_status +2025-01-06 22:22:25,994 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:27,006 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:27,975 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:27,999 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:29,000 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:30,016 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:31,005 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:32,012 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:33,004 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:33,058 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:22:33,058 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:22:33,265 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:22:33,498 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:33,498 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:33,499 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 22:22:33,499 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media +2025-01-06 22:22:33,499 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval +2025-01-06 22:22:33,499 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table +2025-01-06 22:22:33,510 INFO wandb-upload_1:2123564 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphynk_6q6 +2025-01-06 22:22:33,670 INFO wandb-upload_2:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/85pm559e-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 22:22:33,708 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 22:22:33,719 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 22:22:33,747 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/nelgnvoq-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:33,940 INFO SenderThread:2123564 [sender.py:send_request_log_artifact():1525] logged artifact run-917c47lf-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU3Mjg1MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0MDMwNA==', 'latestArtifact': None}} +2025-01-06 22:22:33,940 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:22:33,940 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2025-01-06 22:22:33,941 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:33,941 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:33,941 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,941 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,942 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:33,942 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,946 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,946 DEBUG SenderThread:2123564 [sender.py:send():379] send: telemetry +2025-01-06 22:22:33,946 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,946 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,946 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,946 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,947 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,947 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,947 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,947 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,948 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,948 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,948 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,948 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,948 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,949 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,949 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,949 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,949 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,950 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,950 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,950 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,950 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:33,951 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:33,951 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 22:22:34,004 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:34,189 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/102le9fn-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 22:22:34,288 INFO wandb-upload_0:2123564 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmh_9g91f +2025-01-06 22:22:34,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:34,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:34,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json +2025-01-06 22:22:34,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval +2025-01-06 22:22:34,680 INFO SenderThread:2123564 [sender.py:send_request_log_artifact():1525] logged artifact run-917c47lf-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMjU3Mjg4Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDc0MDMwOQ==', 'latestArtifact': None}} +2025-01-06 22:22:34,680 DEBUG SenderThread:2123564 [sender.py:send():379] send: files +2025-01-06 22:22:34,680 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json with policy now +2025-01-06 22:22:34,680 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:34,680 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:34,681 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:34,852 INFO wandb-upload_2:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/k8zaf9nu-media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json +2025-01-06 22:22:35,004 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:35,094 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/t645rmde-media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json +2025-01-06 22:22:35,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:35,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:36,005 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:36,361 DEBUG SenderThread:2123564 [sender.py:send():379] send: exit +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:send_exit():588] handling runtime: 265 +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:send_exit():594] send defer +2025-01-06 22:22:36,362 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,362 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 22:22:36,362 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 22:22:36,362 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 1 +2025-01-06 22:22:36,363 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,363 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 22:22:36,363 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:36,363 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 22:22:36,363 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 2 +2025-01-06 22:22:36,363 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,363 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 22:22:36,363 INFO HandlerThread:2123564 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 22:22:36,363 DEBUG SystemMonitor:2123564 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 22:22:36,363 DEBUG SystemMonitor:2123564 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 22:22:36,364 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 22:22:36,364 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 22:22:36,414 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 22:22:36,415 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 22:22:36,415 INFO HandlerThread:2123564 [interfaces.py:finish():200] Joined network monitor +2025-01-06 22:22:36,416 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:36,416 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 22:22:36,416 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 3 +2025-01-06 22:22:36,416 DEBUG SenderThread:2123564 [sender.py:send():379] send: stats +2025-01-06 22:22:36,416 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,416 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 22:22:36,417 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:36,417 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 22:22:36,417 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 4 +2025-01-06 22:22:36,417 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:36,417 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 22:22:36,500 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:37,501 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:43,200 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:43,200 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:43,200 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 22:22:43,202 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:43,202 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 22:22:43,202 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 5 +2025-01-06 22:22:43,202 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:43,202 DEBUG SenderThread:2123564 [sender.py:send():379] send: history +2025-01-06 22:22:43,202 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:43,202 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 22:22:43,202 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: summary_record +2025-01-06 22:22:43,203 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:43,203 DEBUG SenderThread:2123564 [sender.py:send():379] send: summary +2025-01-06 22:22:43,207 INFO SenderThread:2123564 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 22:22:43,207 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:43,207 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 22:22:43,207 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 6 +2025-01-06 22:22:43,207 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:43,207 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 22:22:43,208 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:43,208 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 22:22:43,211 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 22:22:43,328 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 7 +2025-01-06 22:22:43,328 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:43,328 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 22:22:43,329 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:43,329 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 22:22:43,363 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:43,503 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml +2025-01-06 22:22:43,503 INFO Thread-12 :2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:46,639 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 8 +2025-01-06 22:22:46,639 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:46,640 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:46,640 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 22:22:46,640 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:46,640 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 22:22:46,640 INFO SenderThread:2123564 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 22:22:46,642 INFO SenderThread:2123564 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 22:22:46,644 INFO SenderThread:2123564 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 22:22:46,657 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 9 +2025-01-06 22:22:46,658 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:46,658 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 22:22:46,658 DEBUG SenderThread:2123564 [sender.py:send():379] send: artifact +2025-01-06 22:22:46,874 INFO SenderThread:2123564 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTk5MTI2Nw==', 'versionIndex': 427}}} +2025-01-06 22:22:46,874 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:46,874 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 22:22:46,874 INFO SenderThread:2123564 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 22:22:47,364 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:47,504 INFO SenderThread:2123564 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:47,505 INFO SenderThread:2123564 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files +2025-01-06 22:22:47,505 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml config.yaml +2025-01-06 22:22:47,505 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json wandb-summary.json +2025-01-06 22:22:47,505 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:47,508 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/requirements.txt requirements.txt +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-metadata.json wandb-metadata.json +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log output.log +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/diff.patch diff.patch +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4 +2025-01-06 22:22:47,509 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4 +2025-01-06 22:22:47,510 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 media/videos/eval/1_eval_1_5a016504166fce504de7.mp4 +2025-01-06 22:22:47,510 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4 +2025-01-06 22:22:47,510 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4 +2025-01-06 22:22:47,512 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2025-01-06 22:22:47,514 INFO SenderThread:2123564 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json +2025-01-06 22:22:47,514 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 10 +2025-01-06 22:22:47,514 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:47,514 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:47,514 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 22:22:47,514 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:47,515 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 22:22:47,515 INFO SenderThread:2123564 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:22:47,681 INFO wandb-upload_2:2123564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml +2025-01-06 22:22:47,698 INFO wandb-upload_1:2123564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json +2025-01-06 22:22:47,735 INFO wandb-upload_3:2123564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/requirements.txt +2025-01-06 22:22:47,742 INFO wandb-upload_0:2123564 [upload_job.py:push():130] Uploaded file /tmp/tmpy2mbo5uowandb/nsvrz20o-events.out.tfevents.1736219890.leonmkim-ROG-Strix-G15CS-G15CS.2123316.0 +2025-01-06 22:22:47,744 INFO wandb-upload_4:2123564 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log +2025-01-06 22:22:47,944 INFO Thread-11 (_thread_body):2123564 [sender.py:transition_state():614] send defer: 11 +2025-01-06 22:22:47,944 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:47,944 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 22:22:47,945 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:47,945 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 22:22:47,945 INFO SenderThread:2123564 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 22:22:47,946 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 12 +2025-01-06 22:22:47,946 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:47,946 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 22:22:47,946 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:47,946 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 22:22:47,946 INFO SenderThread:2123564 [file_stream.py:finish():601] file stream finish called +2025-01-06 22:22:48,007 INFO SenderThread:2123564 [file_stream.py:finish():605] file stream finish is done +2025-01-06 22:22:48,007 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 13 +2025-01-06 22:22:48,007 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:48,007 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 22:22:48,007 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:48,007 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 22:22:48,007 INFO SenderThread:2123564 [sender.py:transition_state():614] send defer: 14 +2025-01-06 22:22:48,007 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: defer +2025-01-06 22:22:48,007 DEBUG SenderThread:2123564 [sender.py:send():379] send: final +2025-01-06 22:22:48,008 INFO HandlerThread:2123564 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 22:22:48,008 DEBUG SenderThread:2123564 [sender.py:send():379] send: footer +2025-01-06 22:22:48,008 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: defer +2025-01-06 22:22:48,008 INFO SenderThread:2123564 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 22:22:48,008 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:48,008 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 22:22:48,009 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:48,009 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 22:22:48,009 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: server_info +2025-01-06 22:22:48,009 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 22:22:48,011 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 22:22:48,012 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 22:22:48,061 DEBUG SenderThread:2123564 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 22:22:48,061 INFO MainThread:2123564 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 22:22:48,062 INFO MainThread:2123564 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 22:22:48,062 INFO MainThread:2123564 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 22:22:48,062 DEBUG HandlerThread:2123564 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 22:22:48,062 INFO HandlerThread:2123564 [handler.py:finish():882] shutting down handler +2025-01-06 22:22:49,010 INFO WriterThread:2123564 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/run-917c47lf.wandb +2025-01-06 22:22:49,061 INFO SenderThread:2123564 [sender.py:finish():1615] shutting down sender +2025-01-06 22:22:49,062 INFO SenderThread:2123564 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 22:22:49,062 INFO SenderThread:2123564 [file_pusher.py:join():175] waiting for file pusher diff --git a/917c47lf/wandb/debug.log b/917c47lf/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..82ad1fc1326b77162e48852746096f1cf7a46859 --- /dev/null +++ b/917c47lf/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Configure stats pid to 2123316 +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/settings +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug.log +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/wandb/run-20250106_221810-917c47lf/logs/debug-internal.log +2025-01-06 22:18:10,152 INFO MainThread:2123316 [wandb_init.py:init():569] calling init triggers +2025-01-06 22:18:10,153 INFO MainThread:2123316 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf'} +2025-01-06 22:18:10,153 INFO MainThread:2123316 [wandb_init.py:init():619] starting backend +2025-01-06 22:18:10,153 INFO MainThread:2123316 [wandb_init.py:init():623] setting up manager +2025-01-06 22:18:10,158 INFO MainThread:2123316 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 22:18:10,159 INFO MainThread:2123316 [wandb_init.py:init():631] backend started and connected +2025-01-06 22:18:10,172 INFO MainThread:2123316 [wandb_init.py:init():720] updated telemetry +2025-01-06 22:18:10,178 INFO MainThread:2123316 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 22:18:10,428 INFO MainThread:2123316 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 22:18:10,494 INFO MainThread:2123316 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 22:18:10,494 INFO MainThread:2123316 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 22:18:10,823 INFO MainThread:2123316 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 22:18:10,823 INFO MainThread:2123316 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 22:18:10,823 INFO MainThread:2123316 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 22:18:10,823 INFO MainThread:2123316 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 22:18:10,825 INFO MainThread:2123316 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 22:18:10,825 INFO MainThread:2123316 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/tb, True +2025-01-06 22:18:17,522 INFO MainThread:2123316 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lady', 'left_book_slot': 'twodim'} +2025-01-06 22:22:49,064 WARNING MsgRouterThr:2123316 [router.py:message_loop():77] message_loop has been closed diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml b/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b262844197a3c6cc4e6dc88984927c5d062a075 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736219890 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lady +left_book_slot: + desc: null + value: twodim diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/diff.patch b/917c47lf/wandb/run-20250106_221810-917c47lf/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c28171b0ccbf1f8e5d0d27b96acf339a03de1c06 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..a719175 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log b/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..15dfb8ddf527f86a3208a8f6ecbc4debfbf13245 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/output.log @@ -0,0 +1,266 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736219897.522906]: resetting environment +[INFO] [1736219897.526029]: cleared current plan +[INFO] [1736219897.527141]: moving to home +[INFO] [1736219898.529342]: reached home +[INFO] [1736219898.530628]: reset action history +[INFO] [1736219901.238600]: environment reset +Starting episode 0 +[INFO] [1736219901.240535]: resetting environment +[INFO] [1736219901.245729]: cleared current plan +[INFO] [1736219901.246829]: moving to home +[INFO] [1736219902.249031]: reached home +[INFO] [1736219902.250479]: reset action history +[INFO] [1736219904.957737]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736219909.970324]: resetting environment +[INFO] [1736219909.975152]: cleared current plan +[INFO] [1736219909.976553]: moving to home +[INFO] [1736219910.978882]: reached home +[INFO] [1736219910.980200]: reset action history +[INFO] [1736219913.688343]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/episode_rosbags/episode_0_2025-01-06-22-18-33.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2123316.log +started roslaunch server http://158.130.50.37:45993/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2123688] +process[print_text-2]: started with pid [2123689] +process[rosbag_record-3]: started with pid [2123690] +started bagging! +[INFO] [1736219940.709040]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736219941.443218]: Stored episode 1. +[INFO] [1736219941.443862]: Saving video... +[INFO] [1736219941.650064]: Video saved! +Starting episode 1 +[INFO] [1736219941.662831]: resetting environment +[INFO] [1736219941.673129]: cleared current plan +[INFO] [1736219941.673894]: moving to home +[INFO] [1736219945.076641]: reached home +[INFO] [1736219945.077085]: reset action history +[INFO] [1736219947.785034]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736219950.631790]: resetting environment +[INFO] [1736219950.635650]: cleared current plan +[INFO] [1736219950.635818]: moving to home +[INFO] [1736219951.637275]: reached home +[INFO] [1736219951.637775]: reset action history +[INFO] [1736219954.345847]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/episode_rosbags/episode_1_2025-01-06-22-19-14.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2123316.log +started roslaunch server http://158.130.50.37:36559/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2123806] +process[print_text-5]: started with pid [2123807] +process[rosbag_record-6]: started with pid [2123831] +started bagging! +For topic gripper_width: timestamp difference is -139318816 for nearest: 1736219973290735849 - target: 1736219973430054665 at idx 249 +[INFO] [1736219981.331279]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736219982.087740]: Stored episode 2. +[INFO] [1736219982.088427]: Saving video... +[INFO] [1736219982.531813]: Video saved! +Starting episode 2 +[INFO] [1736219982.578438]: resetting environment +[INFO] [1736219982.597663]: cleared current plan +[INFO] [1736219982.601182]: moving to home +[INFO] [1736219985.713397]: reached home +[INFO] [1736219985.715861]: reset action history +[INFO] [1736219988.489256]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736219990.581380]: resetting environment +[INFO] [1736219990.590709]: cleared current plan +[INFO] [1736219990.592498]: moving to home +[INFO] [1736219991.594621]: reached home +[INFO] [1736219991.596171]: reset action history +[INFO] [1736219994.375444]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/episode_rosbags/episode_2_2025-01-06-22-19-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2123316.log +started roslaunch server http://158.130.50.37:34043/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2123964] +process[print_text-8]: started with pid [2123965] +process[rosbag_record-9]: started with pid [2123966] +started bagging! +[INFO] [1736220024.100123]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736220024.881471]: Stored episode 3. +[INFO] [1736220024.887772]: Saving video... +[INFO] [1736220025.141884]: Video saved! +Starting episode 3 +[INFO] [1736220025.189429]: resetting environment +[INFO] [1736220025.217827]: cleared current plan +[INFO] [1736220025.218105]: moving to home +[INFO] [1736220028.321397]: reached home +[INFO] [1736220028.322074]: reset action history +[INFO] [1736220031.117002]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220053.257400]: resetting environment +[INFO] [1736220053.283014]: cleared current plan +[INFO] [1736220053.288772]: moving to home +[INFO] [1736220054.294114]: reached home +[INFO] [1736220054.305027]: reset action history +[INFO] [1736220057.098316]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/episode_rosbags/episode_3_2025-01-06-22-20-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2123316.log +started roslaunch server http://158.130.50.37:36945/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2124100] +process[print_text-11]: started with pid [2124101] +process[rosbag_record-12]: started with pid [2124102] +started bagging! +For topic gripper_width: timestamp difference is 112251660 for nearest: 1736220073757399937 - target: 1736220073645148277 at idx 233 +[INFO] [1736220084.528105]: Storing episode... +[print_text-11] killing on exit[pub_text-10] killing on exit +[rosbag_record-12] killing on exit +[INFO] [1736220085.281030]: Stored episode 4. +[INFO] [1736220085.293392]: Saving video... +[INFO] [1736220085.566281]: Video saved! +Starting episode 4 +[INFO] [1736220085.623893]: resetting environment +[INFO] [1736220085.647403]: cleared current plan +[INFO] [1736220085.651819]: moving to home +[INFO] [1736220089.264659]: reached home +[INFO] [1736220089.281027]: reset action history +[INFO] [1736220092.075322]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736220101.544903]: resetting environment +[INFO] [1736220101.563016]: cleared current plan +[INFO] [1736220101.573519]: moving to home +[INFO] [1736220102.582005]: reached home +[INFO] [1736220102.590284]: reset action history +[INFO] [1736220105.383448]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/917c47lf/episode_rosbags/episode_4_2025-01-06-22-21-45.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2123316.log +started roslaunch server http://158.130.50.37:44951/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2124236] +process[print_text-14]: started with pid [2124237] +process[rosbag_record-15]: started with pid [2124238] +started bagging! +For topic gripper_width: timestamp difference is 105098769 for nearest: 1736220115390804097 - target: 1736220115285705328 at idx 132 +For topic gripper_width: timestamp difference is -288806435 for nearest: 1736220129190732244 - target: 1736220129479538679 at idx 249 +[INFO] [1736220133.247903]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit[pub_text-13] killing on exit +[INFO] [1736220134.333839]: Stored episode 5. +[INFO] [1736220134.339141]: Saving video... +[INFO] [1736220134.569069]: Video saved! +[INFO] [1736220134.619328]: resetting environment +[INFO] [1736220134.646660]: cleared current plan +[INFO] [1736220134.654105]: moving to home +[WARN] [1736220134.602562]: Plan exhausted +[WARN] [1736220134.640399]: Plan exhausted +[INFO] [1736220137.860131]: reached home +[INFO] [1736220137.861571]: reset action history +[INFO] [1736220140.736461]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/requirements.txt b/917c47lf/wandb/run-20250106_221810-917c47lf/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-metadata.json b/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f491c07ec0a754a1455810a8166d98e3b4f20ea3 --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T03:18:10.518603", + "startedAt": "2025-01-07T03:18:10.150271", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3562.4985, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.001, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4500.056, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 564.8245735168457 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json b/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..19f7daa5570e28d5412d69f284b76852a8e9679a --- /dev/null +++ b/917c47lf/wandb/run-20250106_221810-917c47lf/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "51de33a49d1b3553fbba41c97b8700d45a16e34447711639790a2881dcf2c6a3", "size": 1188873, "path": "media/videos/eval/0_eval_0_51de33a49d1b3553fbba.mp4"}, "global_step": 665, "_timestamp": 1736220153.5786548, "_runtime": 263.41915488243103, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "5a016504166fce504de7265eb372b780204054bfb5768857df26ed6e1c0d442c", "size": 1189054, "path": "media/videos/eval/1_eval_1_5a016504166fce504de7.mp4"}, "eval/num_success": 1.0, "episode": 25.0, "eval/success_rate": 0.20000000298023224, "eval/success_rate_lb_0.01": 0.0036819279193878174, "eval/success_rate_ub_0.01": 0.7286170721054077, "eval/success_rate_lb_0.025": 0.005147464573383331, "eval/success_rate_ub_0.025": 0.6458297371864319, "eval/success_rate_lb_0.05": 0.0251171737909317, "eval/success_rate_ub_0.05": 0.5745371580123901, "eval/success_rate_lb_0.1": 0.10005546361207962, "eval/success_rate_ub_0.1": 0.47114917635917664, "eval/2_eval": {"_type": "video-file", "sha256": "4e8e215f602a2441e87a870098065953bee6249823d409431e7d40031381d4b3", "size": 1304735, "path": "media/videos/eval/2_eval_2_4e8e215f602a2441e87a.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "4cfc3fdd64623711fec5c4b51ef4a0c2497c8c628ded5e0279dc9eae3cff9f66", "size": 1144448, "path": "media/videos/eval/3_eval_3_4cfc3fdd64623711fec5.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "c3e93442594f3e5f3d1e87d8fea5936129990d994d930dde5165fb385ac3e0a0", "size": 1238915, "path": "media/videos/eval/4_eval_4_c3e93442594f3e5f3d1e.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "ff553b4df26f1e2b1b21e12412af8c8e47b3dc0662f8640d7c2b7298b63d3c98", "size": 405, "artifact_path": "wandb-client-artifact://rpaid5odsbe8mfse513df9driy5tf0zzrkruvpaym97o3pie4xw2cmfo8xm30l73lnk997xqf4sf2d9er8mc6fszpksnt7wugmv8e5jt0bua7lcxrq9tmrioaisxzkvr/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://fle16w9t0cnok980xf1jjqvcv4tsrs0ejyulf9gpy8ls5h2w4gnahswlxhr2sxm8dbhggwk583w41qvgg7irdyrgbcvmlkxrrbrdbtsnu0us720z5va9dncb8ulyn9g2:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 11, "total_num_episodes": 25, "total_success_rate": 0.44, "total_success_rate_lb_0.01": 0.22379118209640997, "total_success_rate_ub_0.01": 0.6720014512194907, "total_success_rate_lb_0.025": 0.2644356638664259, "total_success_rate_ub_0.025": 0.649185761779512, "total_success_rate_lb_0.05": 0.29046089951414444, "total_success_rate_ub_0.05": 0.5987451448839108, "total_success_rate_lb_0.1": 0.3072170541075988, "total_success_rate_ub_0.1": 0.5827415138317564, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "c996a401a7f2ec9cbcab74aca175a9a6e25057d686d48454c8e7c1af8bd08e93", "size": 507, "artifact_path": "wandb-client-artifact://9at61sp0ue2fu0rdhw17hqhi1ll66vx9fajdz75y529n8ebbrcceztf1wyhqwrjp2pijw5iupo8gflw8j9fg6yrgiucr2r6ivger6r657ip8cu8i1y9c6iicrlnhooup/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://i8j87qqwzyhz674ccqnmfizdleifbrjh3kemvxqu0i5xhavl9z19cc8gmzebld7nvzdhmdmygkulu7mo5ltowmk6fb6rcb7mrfj1riidt7h32a61vk6b5ya6dumguojp:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_c996a401a7f2ec9cbcab.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 265}, "eval/total_success_rate": 0.4399999976158142, "eval/total_success_rate_lb_0.01": 0.2237911820411682, "eval/total_success_rate_ub_0.01": 0.6720014214515686, "eval/total_success_rate_lb_0.025": 0.26443567872047424, "eval/total_success_rate_ub_0.025": 0.6491857767105103, "eval/total_success_rate_lb_0.05": 0.2904609143733978, "eval/total_success_rate_ub_0.05": 0.5987451672554016, "eval/total_success_rate_lb_0.1": 0.3072170615196228, "eval/total_success_rate_ub_0.1": 0.5827414989471436} \ No newline at end of file diff --git a/917c47lf/wandb/run-20250106_221810-917c47lf/run-917c47lf.wandb b/917c47lf/wandb/run-20250106_221810-917c47lf/run-917c47lf.wandb new file mode 100644 index 0000000000000000000000000000000000000000..3bda22d9509cb4db5120259ad5d236b70c6b2d00 Binary files /dev/null and b/917c47lf/wandb/run-20250106_221810-917c47lf/run-917c47lf.wandb differ diff --git a/e3yh61zm/episode_rosbags/episode_1_2025-01-07-17-56-35.bag b/e3yh61zm/episode_rosbags/episode_1_2025-01-07-17-56-35.bag new file mode 100644 index 0000000000000000000000000000000000000000..8c1b20f93ff5c308bef636f42e526c651d82028f --- /dev/null +++ b/e3yh61zm/episode_rosbags/episode_1_2025-01-07-17-56-35.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:febb9b0f2ef7023fb7666e101418e9541fbcd73758126a2ff8881f272de5c6a0 +size 1412710766 diff --git a/e3yh61zm/episode_rosbags/episode_2_2025-01-07-17-57-28.bag b/e3yh61zm/episode_rosbags/episode_2_2025-01-07-17-57-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..c15871358fe5a2922cb35782ce495c699d6f3228 --- /dev/null +++ b/e3yh61zm/episode_rosbags/episode_2_2025-01-07-17-57-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:069d010bdef3c90df6fcb2ca08632f2351192ff86c2830e5b701e66b97cda935 +size 1414149100 diff --git a/i3pbiz9p/.hydra/config.yaml b/i3pbiz9p/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/i3pbiz9p/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/i3pbiz9p/.hydra/hydra.yaml b/i3pbiz9p/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34a885a9c27392dcff56f3d15ddf58b8b7ecc087 --- /dev/null +++ b/i3pbiz9p/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/i3pbiz9p/.hydra/overrides.yaml b/i3pbiz9p/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/i3pbiz9p/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/i3pbiz9p/episode_rosbags/aligned_depth_to_color_K.npy b/i3pbiz9p/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/i3pbiz9p/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/i3pbiz9p/episode_rosbags/cam_tf_world.npy b/i3pbiz9p/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/i3pbiz9p/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/i3pbiz9p/episode_rosbags/color_K.npy b/i3pbiz9p/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/i3pbiz9p/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/i3pbiz9p/episode_rosbags/depth_K.npy b/i3pbiz9p/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/i3pbiz9p/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/i3pbiz9p/eval_robot.log b/i3pbiz9p/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..bea66db89cf203cf638891f46197dd1799837cd3 --- /dev/null +++ b/i3pbiz9p/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-06 17:33:30,785][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 17:33:30,787][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/i3pbiz9p/eval_video/0_eval.mp4 b/i3pbiz9p/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a07860326342abe6b448ffb60304914b9a55b8ba --- /dev/null +++ b/i3pbiz9p/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f6567d85218e9cc2a5076bccfe7c4a80a000044f84890f5a1722a66f128d7bb +size 1223582 diff --git a/i3pbiz9p/eval_video/1_eval.mp4 b/i3pbiz9p/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e5b8aad901f10c656f303bad914cb54ed8fc2dd5 --- /dev/null +++ b/i3pbiz9p/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54887f1093ebbceb175ef9c830cac8037076d468a4f6bfc5ee4ab454b9f6e84c +size 1257490 diff --git a/i3pbiz9p/eval_video/2_eval.mp4 b/i3pbiz9p/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ed15d944cd652aaaede466ece3e0d2259e34b08c --- /dev/null +++ b/i3pbiz9p/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f1a4ab77045217b2016fb14d3fa82236d824a2bff91323047cd1e56d38a2ea0 +size 1214564 diff --git a/i3pbiz9p/eval_video/3_eval.mp4 b/i3pbiz9p/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..90c06be2bc6d87662ec510c2f27811c4ee15d3c3 --- /dev/null +++ b/i3pbiz9p/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd256e78c90500bf4c0d3d7238228f563d0f8bce065c57c00c528115140b02bf +size 1263661 diff --git a/i3pbiz9p/eval_video/4_eval.mp4 b/i3pbiz9p/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2c54a1e0f2807e3d8db62820996d6ddd3e23bafc --- /dev/null +++ b/i3pbiz9p/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:498f7f4f2ba24e669402c9dfec977c5b708a23374ae6ca9484423ee7deb31123 +size 1224941 diff --git a/i3pbiz9p/tb/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 b/i3pbiz9p/tb/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 new file mode 100644 index 0000000000000000000000000000000000000000..fc80f429867f9f74ed479747493328e6c8ba28be --- /dev/null +++ b/i3pbiz9p/tb/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:039c0e4537eaa79ec62fb9752c0c3b7b5e0ba231316bdb7f839bb122906d6276 +size 6539 diff --git a/i3pbiz9p/wandb/debug-internal.log b/i3pbiz9p/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d5d9fc363e7d6c1b5ce780068ffd220a9a21c5a2 --- /dev/null +++ b/i3pbiz9p/wandb/debug-internal.log @@ -0,0 +1,804 @@ +2025-01-06 17:33:36,739 INFO StreamThr :2091683 [internal.py:wandb_internal():85] W&B internal server running at pid: 2091683, started at: 2025-01-06 17:33:36.739296 +2025-01-06 17:33:36,740 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:33:36,751 INFO WriterThread:2091683 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/run-i3pbiz9p.wandb +2025-01-06 17:33:36,751 DEBUG SenderThread:2091683 [sender.py:send():379] send: header +2025-01-06 17:33:36,760 DEBUG SenderThread:2091683 [sender.py:send():379] send: run +2025-01-06 17:33:37,035 INFO SenderThread:2091683 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files +2025-01-06 17:33:37,035 INFO SenderThread:2091683 [sender.py:_start_run_threads():1188] run started: i3pbiz9p with start time 1736202816.739327 +2025-01-06 17:33:37,051 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:33:37,051 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:33:37,094 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:33:37,099 DEBUG HandlerThread:2091683 [system_info.py:__init__():26] System info init +2025-01-06 17:33:37,099 DEBUG HandlerThread:2091683 [system_info.py:__init__():41] System info init done +2025-01-06 17:33:37,099 INFO HandlerThread:2091683 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:33:37,099 INFO SystemMonitor:2091683 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:33:37,100 INFO HandlerThread:2091683 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:33:37,100 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:33:37,101 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:33:37,101 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:33:37,102 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:33:37,102 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:33:37,112 DEBUG HandlerThread:2091683 [system_info.py:probe():152] Probing system +2025-01-06 17:33:37,114 DEBUG HandlerThread:2091683 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:33:37,118 DEBUG HandlerThread:2091683 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:33:37,119 DEBUG HandlerThread:2091683 [system_info.py:probe():200] Probing system done +2025-01-06 17:33:37,119 DEBUG HandlerThread:2091683 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:33:37.112692', 'startedAt': '2025-01-06T22:33:36.729865', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.9458750000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.97, 'min': 800.0, 'max': 4700.0}, {'current': 4495.522, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.979, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 346.4975051879883}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:33:37,119 INFO HandlerThread:2091683 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:33:37,119 INFO HandlerThread:2091683 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:33:37,119 DEBUG HandlerThread:2091683 [system_info.py:_save_code():44] Saving code +2025-01-06 17:33:37,120 DEBUG HandlerThread:2091683 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:33:37,120 DEBUG HandlerThread:2091683 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:33:37,182 DEBUG HandlerThread:2091683 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:33:37,182 INFO HandlerThread:2091683 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:33:37,185 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:33:37,185 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:33:37,186 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:33:37,186 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:33:37,429 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/d1l9j1pm-wandb-metadata.json +2025-01-06 17:33:37,435 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:33:37,435 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:33:37,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:33:37,436 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:33:37,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:37,481 INFO HandlerThread:2091683 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/tb" + save: true +} +control { + relay_id: "1924260d83af496aa73763b17d76d970" +} +_info { + stream_id: "i3pbiz9p" +} + +2025-01-06 17:33:37,481 DEBUG HandlerThread:2091683 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:33:37,482 INFO wandb-upload_1:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/02d26m1c-code/FISH/eval_robot.py +2025-01-06 17:33:37,499 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/3at6l5x8-diff.patch +2025-01-06 17:33:37,556 DEBUG SenderThread:2091683 [sender.py:send():379] send: telemetry +2025-01-06 17:33:37,556 DEBUG SenderThread:2091683 [sender.py:send():379] send: tbrecord +2025-01-06 17:33:37,557 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:33:37,557 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 with policy live +2025-01-06 17:33:37,751 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/360bta9l-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:33:38,037 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-metadata.json +2025-01-06 17:33:38,037 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/requirements.txt +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code/FISH/eval_robot.py +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/diff.patch +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code/FISH +2025-01-06 17:33:38,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:39,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:40,036 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:40,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:41,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:42,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:42,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:43,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:44,335 DEBUG SenderThread:2091683 [sender.py:send():379] send: config +2025-01-06 17:33:44,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:45,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:46,037 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:46,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:47,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:47,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:48,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:48,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:49,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:50,039 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:50,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:51,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:52,039 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:52,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:33:52,436 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:33:52,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:52,590 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:53,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:54,040 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:54,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:55,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:56,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:57,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:57,968 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:58,041 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:58,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:59,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:00,042 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:00,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:01,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:02,042 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:02,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:03,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:03,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:04,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:05,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:06,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:07,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:34:07,436 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:34:07,477 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:08,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:08,543 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:09,044 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml +2025-01-06 17:34:09,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:10,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:11,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:12,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:13,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:14,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:14,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:15,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:16,046 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:16,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:17,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:18,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:19,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:19,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:20,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:21,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:22,048 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:22,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:34:22,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:34:22,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:23,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:24,048 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:24,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:25,373 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:25,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:26,049 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:26,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:27,444 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:28,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:28,671 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:34:28,671 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 with policy now +2025-01-06 17:34:28,672 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:34:28,673 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:34:28,673 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:34:28,674 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:34:28,931 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/t0898jau-media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos +2025-01-06 17:34:29,337 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/49fouzfx-media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 +2025-01-06 17:34:29,444 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:30,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:30,444 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:30,678 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:31,444 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:32,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:33,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:34,051 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:34,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:35,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:36,053 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:36,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:36,637 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:37,103 DEBUG SystemMonitor:2091683 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:34:37,103 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:34:37,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:34:37,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:34:37,477 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:38,054 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:34:38,306 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/7gazhosy-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:34:38,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:39,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:40,055 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:40,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:41,446 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:42,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:42,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:43,446 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:44,056 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:44,447 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:45,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:46,447 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:47,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:47,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:48,447 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:49,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:50,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:51,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:52,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:34:52,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:34:52,477 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:52,588 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:53,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:54,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:55,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:56,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:57,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:58,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:58,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:59,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:00,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:01,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:02,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:03,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:03,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:04,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:05,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:06,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:07,111 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:35:07,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:35:07,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:35:07,477 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:08,450 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:08,968 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:09,371 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:35:09,371 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 with policy now +2025-01-06 17:35:09,371 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:35:09,372 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:35:09,372 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:35:09,372 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:35:09,450 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:09,628 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/v6xtb6yv-media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 +2025-01-06 17:35:10,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:35:10,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:10,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 +2025-01-06 17:35:10,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:35:10,324 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/lpqadr1r-media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 +2025-01-06 17:35:10,451 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:11,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:12,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:12,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:13,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:14,066 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:14,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:14,729 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:15,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:16,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:17,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:18,068 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:18,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:19,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:20,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:20,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:21,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:22,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:35:22,438 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:35:22,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:23,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:24,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:25,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:25,591 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:26,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:27,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:28,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:29,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:30,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:31,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:31,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:32,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:33,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:34,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:35,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:36,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:37,104 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:35:37,106 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:37,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:35:37,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:35:37,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:38,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:39,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:40,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:41,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:42,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:42,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:43,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:44,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:45,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:46,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:47,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:48,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:48,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:49,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:50,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:51,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:52,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:35:52,438 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:35:52,485 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:53,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:53,548 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:54,080 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:35:54,328 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/cywpf5cw-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:35:54,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:54,930 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:35:54,930 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:35:54,932 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:35:55,080 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:35:55,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:56,080 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:56,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:57,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:58,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:58,553 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:59,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:00,082 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:00,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:01,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:02,082 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:02,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:03,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:04,083 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:04,320 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:04,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:05,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:06,084 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:06,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:07,105 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:36:07,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:36:07,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:36:07,485 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:08,085 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:08,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:09,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:09,513 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:10,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:11,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:12,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:13,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:14,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:15,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:15,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:16,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:17,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:18,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:19,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:20,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:21,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:21,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:22,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:36:22,438 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:36:22,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:23,462 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:24,462 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:25,464 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:25,833 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:36:25,834 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:36:25,834 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 with policy now +2025-01-06 17:36:25,834 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:36:25,834 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:36:25,834 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:36:26,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:36:26,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:26,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 +2025-01-06 17:36:26,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:36:26,095 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/h1ny00v3-media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 +2025-01-06 17:36:26,359 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/0f2k6ehu-media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 +2025-01-06 17:36:26,468 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:26,835 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:27,466 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:28,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:28,468 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:29,466 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:30,092 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:30,469 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:31,465 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:32,022 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:32,093 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:32,464 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:33,472 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:34,093 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:34,467 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:34,747 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:36:34,747 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:36:34,748 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:36:35,094 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:36:35,094 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:36:35,466 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:36,094 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:36,468 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:37,105 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:36:37,105 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:37,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:36:37,441 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:36:37,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:38,095 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:38,507 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:39,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:40,096 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:40,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:41,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:42,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:42,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:43,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:44,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:45,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:46,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:47,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:47,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:48,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:49,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:50,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:51,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:52,100 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:52,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:36:52,438 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:36:52,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:52,518 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:53,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:54,100 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:54,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:55,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:56,101 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:56,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:57,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:57,549 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:58,102 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:58,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:59,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:00,103 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:00,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:01,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:02,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:03,446 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:03,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:04,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:05,533 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:06,505 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:06,849 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:37:06,850 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:37:06,850 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 with policy now +2025-01-06 17:37:06,851 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:37:06,851 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:37:06,851 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:37:07,103 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/qsdisp5l-media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 +2025-01-06 17:37:07,105 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:37:07,106 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 +2025-01-06 17:37:07,106 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:37:07,106 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:37:07,378 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/l92u06nl-media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 +2025-01-06 17:37:07,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:37:07,442 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:37:07,506 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:08,106 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:08,510 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:08,531 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:09,106 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:09,509 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:10,518 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:11,520 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:12,107 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:12,524 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:13,520 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:14,081 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:14,108 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:14,521 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:15,525 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:16,526 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:17,527 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:18,527 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:19,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:19,529 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:20,527 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:21,528 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:22,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:37:22,445 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:37:22,526 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:23,529 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:24,526 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:24,528 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:25,528 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:26,529 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:27,538 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:28,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:29,533 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:30,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:30,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:31,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:31,814 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:37:31,814 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:37:31,814 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:37:32,114 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:37:32,114 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:37:32,356 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/cjf8a6zg-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:37:32,533 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:33,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:34,115 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:34,537 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:35,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:35,561 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:36,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:37,106 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:37:37,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:37:37,440 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:37:37,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:38,116 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:38,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:39,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:40,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:40,562 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:41,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:42,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:43,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:44,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:45,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:46,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:46,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:47,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:48,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:49,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:50,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:51,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:52,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:37:52,441 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:37:52,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:52,591 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:53,537 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:54,537 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:55,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:56,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:57,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:58,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:58,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:59,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:00,548 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:01,548 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:02,124 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:02,567 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:03,559 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:03,724 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:03,902 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:38:03,902 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 with policy now +2025-01-06 17:38:03,905 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:38:03,905 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:03,905 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:03,911 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:04,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:04,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:04,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 +2025-01-06 17:38:04,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:38:04,164 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/t4rq8q6d-media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 +2025-01-06 17:38:04,429 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/mrycdmvc-media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 +2025-01-06 17:38:04,557 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:05,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:05,560 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:06,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:06,559 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:07,106 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:38:07,485 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:38:07,485 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:38:07,563 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:08,562 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:09,560 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:09,599 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:10,127 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:10,563 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:11,562 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:12,128 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:12,580 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:13,570 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:14,570 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:15,195 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:15,567 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:15,619 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:38:15,619 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:38:15,794 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:38:16,096 INFO wandb-upload_0:2091683 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgtowj55l +2025-01-06 17:38:16,129 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:16,129 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:38:16,129 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:38:16,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media +2025-01-06 17:38:16,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table +2025-01-06 17:38:16,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval +2025-01-06 17:38:16,173 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:38:16,184 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:38:16,565 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:16,599 INFO SenderThread:2091683 [sender.py:send_request_log_artifact():1525] logged artifact run-i3pbiz9p-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk0Mzk0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMzMxNQ==', 'latestArtifact': None}} +2025-01-06 17:38:16,599 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:38:16,599 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-06 17:38:16,599 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:16,599 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,600 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,600 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:16,600 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,604 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,604 DEBUG SenderThread:2091683 [sender.py:send():379] send: telemetry +2025-01-06 17:38:16,604 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,604 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,605 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,605 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,605 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,606 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,606 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,606 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,606 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,607 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,607 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,607 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,607 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,608 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,608 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,608 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,608 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,609 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,609 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,609 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,609 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,610 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,610 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:38:16,808 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/mhk63r2c-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:38:17,051 INFO wandb-upload_2:2091683 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe9w99qyq +2025-01-06 17:38:17,105 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/zu2qupoz-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:38:17,129 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:17,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2025-01-06 17:38:17,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval +2025-01-06 17:38:17,427 INFO SenderThread:2091683 [sender.py:send_request_log_artifact():1525] logged artifact run-i3pbiz9p-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk0Mzk2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMzMyMA==', 'latestArtifact': None}} +2025-01-06 17:38:17,427 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:38:17,428 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json with policy now +2025-01-06 17:38:17,428 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:17,428 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:17,428 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:17,565 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:17,611 INFO wandb-upload_1:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/ols7cqzq-media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2025-01-06 17:38:17,856 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/l5pgckog-media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2025-01-06 17:38:18,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:18,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:18,565 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:19,566 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:20,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:20,566 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:21,566 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:22,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:38:22,442 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:38:22,566 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:23,567 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:24,101 DEBUG SenderThread:2091683 [sender.py:send():379] send: exit +2025-01-06 17:38:24,101 INFO SenderThread:2091683 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:38:24,102 INFO SenderThread:2091683 [sender.py:send_exit():588] handling runtime: 287 +2025-01-06 17:38:24,104 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:24,104 INFO SenderThread:2091683 [sender.py:send_exit():594] send defer +2025-01-06 17:38:24,104 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,104 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:38:24,104 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:24,105 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:38:24,105 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:38:24,105 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,105 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:38:24,105 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:24,105 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:38:24,105 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:38:24,105 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,105 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:38:24,105 INFO HandlerThread:2091683 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:38:24,106 DEBUG SystemMonitor:2091683 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:38:24,106 DEBUG SystemMonitor:2091683 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:38:24,106 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:38:24,107 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:38:24,133 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:24,160 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:38:24,160 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:38:24,160 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:38:24,161 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:24,161 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:38:24,161 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:38:24,161 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:38:24,161 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,161 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:38:24,161 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:24,161 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:38:24,161 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:38:24,161 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,161 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:38:26,133 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:30,872 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:30,872 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:30,872 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:38:30,873 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:30,873 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:38:30,873 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:38:30,873 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:30,873 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:30,873 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:30,873 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:30,874 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:38:30,874 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:30,874 DEBUG SenderThread:2091683 [sender.py:send():379] send: summary +2025-01-06 17:38:30,878 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:30,878 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:30,878 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:38:30,878 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:38:30,878 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:30,879 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:38:30,879 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:30,879 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:38:30,882 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:31,003 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:38:31,003 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:31,003 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:38:31,004 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:31,004 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:38:31,103 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:31,135 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml +2025-01-06 17:38:31,136 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:32,931 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:38:32,931 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:32,932 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:32,932 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:38:32,932 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:32,932 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:38:32,932 INFO SenderThread:2091683 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:38:32,935 INFO SenderThread:2091683 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:38:32,937 INFO SenderThread:2091683 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:38:32,958 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:38:32,958 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:32,958 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:38:32,959 DEBUG SenderThread:2091683 [sender.py:send():379] send: artifact +2025-01-06 17:38:33,103 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:33,136 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:33,188 INFO SenderThread:2091683 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'versionIndex': 424}}} +2025-01-06 17:38:33,188 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:33,188 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:38:33,188 INFO SenderThread:2091683 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:38:34,137 INFO SenderThread:2091683 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files +2025-01-06 17:38:34,137 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml config.yaml +2025-01-06 17:38:34,137 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json wandb-summary.json +2025-01-06 17:38:34,137 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:38:34,138 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/requirements.txt requirements.txt +2025-01-06 17:38:34,138 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:38:34,138 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log output.log +2025-01-06 17:38:34,138 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/diff.patch diff.patch +2025-01-06 17:38:34,141 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:38:34,141 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 +2025-01-06 17:38:34,142 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 +2025-01-06 17:38:34,142 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 +2025-01-06 17:38:34,143 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 +2025-01-06 17:38:34,143 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 +2025-01-06 17:38:34,146 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2025-01-06 17:38:34,146 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:38:34,146 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:38:34,146 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:34,146 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,146 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:38:34,147 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,147 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:38:34,147 INFO SenderThread:2091683 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:38:34,316 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:34,332 INFO wandb-upload_1:2091683 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml +2025-01-06 17:38:34,398 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:34,420 INFO wandb-upload_3:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/urq8oc1x-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:38:34,423 INFO wandb-upload_4:2091683 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/requirements.txt +2025-01-06 17:38:34,623 INFO Thread-11 (_thread_body):2091683 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:38:34,623 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,623 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:38:34,624 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,624 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:38:34,624 INFO SenderThread:2091683 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:38:34,625 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:38:34,625 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,625 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:38:34,625 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,625 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:38:34,625 INFO SenderThread:2091683 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:38:34,677 INFO SenderThread:2091683 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:38:34,677 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:38:34,677 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,677 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:38:34,678 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,678 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:38:34,678 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:38:34,678 DEBUG SenderThread:2091683 [sender.py:send():379] send: final +2025-01-06 17:38:34,678 DEBUG SenderThread:2091683 [sender.py:send():379] send: footer +2025-01-06 17:38:34,678 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,678 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:38:34,678 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,678 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:38:34,679 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:34,679 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:34,679 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:38:34,680 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:34,680 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:38:34,681 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:38:34,682 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:38:34,682 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:34,743 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:34,744 INFO MainThread:2091683 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:38:34,744 INFO MainThread:2091683 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:38:34,744 INFO MainThread:2091683 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:38:34,744 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:38:34,744 INFO HandlerThread:2091683 [handler.py:finish():882] shutting down handler +2025-01-06 17:38:35,680 INFO WriterThread:2091683 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/run-i3pbiz9p.wandb +2025-01-06 17:38:35,743 INFO SenderThread:2091683 [sender.py:finish():1615] shutting down sender +2025-01-06 17:38:35,744 INFO SenderThread:2091683 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:38:35,744 INFO SenderThread:2091683 [file_pusher.py:join():175] waiting for file pusher diff --git a/i3pbiz9p/wandb/debug.log b/i3pbiz9p/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9bc20b05ab5ea3ad1f6e5ebe7d2744b094bc4291 --- /dev/null +++ b/i3pbiz9p/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Configure stats pid to 2091517 +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/settings +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug.log +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug-internal.log +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p'} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:init():619] starting backend +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:init():623] setting up manager +2025-01-06 17:33:36,738 INFO MainThread:2091517 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:33:36,739 INFO MainThread:2091517 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:33:36,750 INFO MainThread:2091517 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:33:36,758 INFO MainThread:2091517 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:33:37,050 INFO MainThread:2091517 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:33:37,090 INFO MainThread:2091517 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:33:37,090 INFO MainThread:2091517 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:33:37,436 INFO MainThread:2091517 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:33:37,436 INFO MainThread:2091517 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:33:37,437 INFO MainThread:2091517 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:33:37,437 INFO MainThread:2091517 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:33:37,440 INFO MainThread:2091517 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:33:37,441 INFO MainThread:2091517 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/tb, True +2025-01-06 17:33:44,335 INFO MainThread:2091517 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:38:35,745 WARNING MsgRouterThr:2091517 [router.py:message_loop():77] message_loop has been closed diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code/FISH/eval_robot.py b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54a10a44eb634c8edb808d263ff678180092a158 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736202816 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: greece +left_book_slot: + desc: null + value: twodim diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/diff.patch b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..c28171b0ccbf1f8e5d0d27b96acf339a03de1c06 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/diff.patch @@ -0,0 +1,63 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index d2afe29..a719175 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -134,11 +134,18 @@ load_checkpoint: ${agent.load_checkpoint} + # all books, 10/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1002_0' ++# wandb_run_id: '1009_1' # seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 ++ ++# RGBD+mask+contact(all ftrs)+act history ++# wandb_run_id: '1003_0' ++# wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history + # wandb_run_id: '1007_0' +-wandb_run_id: '1922_1' # seed 1 ++# wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1938_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history +@@ -146,11 +153,11 @@ wandb_run_id: '1922_1' # seed 1 + # wandb_run_id: '1926_1' # seed 1 + # wandb_run_id: '1930_0' # dataset shuffle seed 1 + +- +- + # RGBD+mask+contact(contact+DTC)+act history + # wandb_run_id: '1745_0' + ++# all books, 4/20 demos per book ++ + # 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act + # 14 demos + +diff --git a/FISH/download_model_checkpoints.py b/FISH/download_model_checkpoints.py +index c7eadce..e3c7dc1 100644 +--- a/FISH/download_model_checkpoints.py ++++ b/FISH/download_model_checkpoints.py +@@ -161,8 +161,18 @@ run_id_list = [ + run_id_list = [ + # '1922_1', + # '1922_2', +- '1926_1', +- '1930_0', ++ # '1944_0', ++ # '1972_0', ++ # '1976_1', ++ # '1948_0', ++ # '1950_0', ++ # '1954_1', ++ '1002_0', ++ '1009_1', ++ '1022_0', ++ '1003_0', ++ '1013_1', ++ '1017_0', + ] + + checkpoint_epoch = 12000 diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json new file mode 100644 index 0000000000000000000000000000000000000000..0d31315a6b7135ea112eda24374d0bc4e7305250 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5290328860224832, 0, 0.4204478860060532, 0, 0.33451344076128464, 0.009447986358166901, 0.2066487521519853]]} \ No newline at end of file diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json new file mode 100644 index 0000000000000000000000000000000000000000..e69fc5d03f73383d42d59221f6b82c19b47fac4f --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[0, 10, 0.0, 0, 0.3317304849961171, 0, 0.3041884303484581, 0, 0.1718106568516029, 0, 0.19463173306135229]]} \ No newline at end of file diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a07860326342abe6b448ffb60304914b9a55b8ba --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f6567d85218e9cc2a5076bccfe7c4a80a000044f84890f5a1722a66f128d7bb +size 1223582 diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e5b8aad901f10c656f303bad914cb54ed8fc2dd5 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:54887f1093ebbceb175ef9c830cac8037076d468a4f6bfc5ee4ab454b9f6e84c +size 1257490 diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ed15d944cd652aaaede466ece3e0d2259e34b08c --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f1a4ab77045217b2016fb14d3fa82236d824a2bff91323047cd1e56d38a2ea0 +size 1214564 diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..90c06be2bc6d87662ec510c2f27811c4ee15d3c3 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd256e78c90500bf4c0d3d7238228f563d0f8bce065c57c00c528115140b02bf +size 1263661 diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2c54a1e0f2807e3d8db62820996d6ddd3e23bafc --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:498f7f4f2ba24e669402c9dfec977c5b708a23374ae6ca9484423ee7deb31123 +size 1224941 diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ed803c48f981184b2b8f6860cc8802292b25a032 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log @@ -0,0 +1,287 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736202824.335711]: resetting environment +[INFO] [1736202824.341810]: cleared current plan +[INFO] [1736202824.343086]: moving to home +[INFO] [1736202825.345111]: reached home +[INFO] [1736202825.346344]: reset action history +[INFO] [1736202828.054907]: environment reset +Starting episode 0 +[INFO] [1736202828.056837]: resetting environment +[INFO] [1736202828.059662]: cleared current plan +[INFO] [1736202828.060566]: moving to home +[INFO] [1736202829.064407]: reached home +[INFO] [1736202829.065839]: reset action history +[INFO] [1736202831.773805]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736202836.959231]: resetting environment +[INFO] [1736202836.964822]: cleared current plan +[INFO] [1736202836.965800]: moving to home +[INFO] [1736202837.968264]: reached home +[INFO] [1736202837.970267]: reset action history +[INFO] [1736202840.678280]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/episode_rosbags/episode_0_2025-01-06-17-34-00.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2091517.log +started roslaunch server http://158.130.50.37:33769/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2091785] +process[print_text-2]: started with pid [2091809] +process[rosbag_record-3]: started with pid [2091810] +started bagging! +For topic gripper_width: timestamp difference is -113159405 for nearest: 1736202854340123905 - target: 1736202854453283310 at idx 187 +For topic gripper_width: timestamp difference is -112756185 for nearest: 1736202854540123768 - target: 1736202854652879953 at idx 188 +For topic gripper_width: timestamp difference is -117601117 for nearest: 1736202860740122834 - target: 1736202860857723951 at idx 249 +For topic gripper_width: timestamp difference is -116787735 for nearest: 1736202861140122589 - target: 1736202861256910324 at idx 249 +For topic gripper_width: timestamp difference is -112664611 for nearest: 1736202863140152305 - target: 1736202863252816916 at idx 249 +For topic gripper_width: timestamp difference is -104865051 for nearest: 1736202866940122389 - target: 1736202867044987440 at idx 249 +[INFO] [1736202867.698122]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736202868.448202]: Stored episode 1. +[INFO] [1736202868.448393]: Saving video... +[INFO] [1736202868.658360]: Video saved! +Starting episode 1 +[INFO] [1736202868.672515]: resetting environment +[INFO] [1736202868.677601]: cleared current plan +[INFO] [1736202868.677860]: moving to home +[INFO] [1736202871.880194]: reached home +[INFO] [1736202871.880753]: reset action history +[INFO] [1736202874.593244]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202877.524156]: resetting environment +[INFO] [1736202877.528202]: cleared current plan +[INFO] [1736202877.528366]: moving to home +[INFO] [1736202878.529328]: reached home +[INFO] [1736202878.529728]: reset action history +[INFO] [1736202881.236484]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/episode_rosbags/episode_1_2025-01-06-17-34-41.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2091517.log +started roslaunch server http://158.130.50.37:35465/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2091920] +process[print_text-5]: started with pid [2091921] +process[rosbag_record-6]: started with pid [2091945] +started bagging! +[INFO] [1736202908.202230]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit[pub_text-4] killing on exit +[INFO] [1736202908.967658]: Stored episode 2. +[INFO] [1736202908.968431]: Saving video... +[INFO] [1736202909.350836]: Video saved! +Starting episode 2 +[INFO] [1736202909.372766]: resetting environment +[INFO] [1736202909.381497]: cleared current plan +[INFO] [1736202909.381757]: moving to home +[INFO] [1736202912.686131]: reached home +[INFO] [1736202912.690334]: reset action history +[INFO] [1736202915.464193]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202953.804082]: resetting environment +[INFO] [1736202953.810292]: cleared current plan +[INFO] [1736202953.810522]: moving to home +[INFO] [1736202954.812117]: reached home +[INFO] [1736202954.812695]: reset action history +[INFO] [1736202957.518115]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/episode_rosbags/episode_2_2025-01-06-17-35-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2091517.log +started roslaunch server http://158.130.50.37:46157/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2092073] +process[print_text-8]: started with pid [2092074] +process[rosbag_record-9]: started with pid [2092098] +started bagging! +For topic gripper_width: timestamp difference is 105284566 for nearest: 1736202958506786460 - target: 1736202958401501894 at idx 15 +For topic gripper_width: timestamp difference is 103778314 for nearest: 1736202959306816530 - target: 1736202959203038216 at idx 28 +For topic gripper_width: timestamp difference is 105002781 for nearest: 1736202960906788965 - target: 1736202960801786184 at idx 43 +For topic gripper_width: timestamp difference is 105192588 for nearest: 1736202961106819556 - target: 1736202961001626968 at idx 45 +For topic gripper_width: timestamp difference is 105642752 for nearest: 1736202961706789688 - target: 1736202961601146936 at idx 52 +For topic gripper_width: timestamp difference is 107106864 for nearest: 1736202963506793201 - target: 1736202963399686337 at idx 71 +For topic gripper_width: timestamp difference is 107714311 for nearest: 1736202964306743280 - target: 1736202964199028969 at idx 81 +For topic gripper_width: timestamp difference is 109056615 for nearest: 1736202965906739377 - target: 1736202965797682762 at idx 103 +[INFO] [1736202984.491730]: Storing episode... +[print_text-8] killing on exit[pub_text-7] killing on exit[rosbag_record-9] killing on exit +[INFO] [1736202985.283368]: Stored episode 3. +[INFO] [1736202985.291994]: Saving video... +[INFO] [1736202985.789465]: Video saved! +Starting episode 3 +[INFO] [1736202985.818721]: resetting environment +[INFO] [1736202985.825536]: cleared current plan +[INFO] [1736202985.825750]: moving to home +[INFO] [1736202989.228745]: reached home +[INFO] [1736202989.232581]: reset action history +[INFO] [1736202992.021291]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736202994.629376]: resetting environment +[INFO] [1736202994.635067]: cleared current plan +[INFO] [1736202994.635272]: moving to home +[INFO] [1736202995.636908]: reached home +[INFO] [1736202995.640787]: reset action history +[INFO] [1736202998.514477]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/episode_rosbags/episode_3_2025-01-06-17-36-38.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2091517.log +started roslaunch server http://158.130.50.37:34547/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2092207] +process[print_text-11]: started with pid [2092208] +process[rosbag_record-12]: started with pid [2092232] +started bagging! +For topic gripper_width: timestamp difference is 106165710 for nearest: 1736203010706746164 - target: 1736203010600580454 at idx 153 +For topic gripper_width: timestamp difference is 111407139 for nearest: 1736203012306788065 - target: 1736203012195380926 at idx 174 +For topic gripper_width: timestamp difference is 119313020 for nearest: 1736203014706786412 - target: 1736203014587473392 at idx 206 +For topic gripper_width: timestamp difference is 117191981 for nearest: 1736203016506786297 - target: 1736203016389594316 at idx 233 +For topic gripper_width: timestamp difference is 118775197 for nearest: 1736203017306744643 - target: 1736203017187969446 at idx 244 +For topic gripper_width: timestamp difference is 119246140 for nearest: 1736203017506811991 - target: 1736203017387565851 at idx 246 +[INFO] [1736203025.486750]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736203026.250851]: Stored episode 4. +[INFO] [1736203026.253724]: Saving video... +[INFO] [1736203026.805652]: Video saved! +Starting episode 4 +[INFO] [1736203026.851493]: resetting environment +[INFO] [1736203026.876003]: cleared current plan +[INFO] [1736203026.877409]: moving to home +[WARN] [1736203026.826584]: Plan exhausted +[WARN] [1736203026.868960]: Plan exhausted +[INFO] [1736203030.286713]: reached home +[INFO] [1736203030.293560]: reset action history +[INFO] [1736203033.036617]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736203051.657182]: resetting environment +[INFO] [1736203051.663651]: cleared current plan +[INFO] [1736203051.663960]: moving to home +[INFO] [1736203052.667361]: reached home +[INFO] [1736203052.668272]: reset action history +[INFO] [1736203055.386809]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/episode_rosbags/episode_4_2025-01-06-17-37-35.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2091517.log +started roslaunch server http://158.130.50.37:41531/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2092344] +process[print_text-14]: started with pid [2092345] +process[rosbag_record-15]: started with pid [2092369] +started bagging! +For topic gripper_width: timestamp difference is 115912876 for nearest: 1736203056806789598 - target: 1736203056690876722 at idx 19 +For topic gripper_width: timestamp difference is 100969467 for nearest: 1736203079606786165 - target: 1736203079505816698 at idx 249 +[INFO] [1736203082.379199]: Storing episode... +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit[print_text-14] killing on exit +[INFO] [1736203083.242451]: Stored episode 5. +[INFO] [1736203083.250481]: Saving video... +[WARN] [1736203083.706614]: Plan exhausted +[WARN] [1736203083.746551]: Plan exhausted +[WARN] [1736203083.789421]: Plan exhausted +[WARN] [1736203083.830040]: Plan exhausted +[WARN] [1736203083.866532]: Plan exhausted +[WARN] [1736203083.912224]: Plan exhausted +[INFO] [1736203083.848782]: Video saved! +[INFO] [1736203083.904201]: resetting environment +[INFO] [1736203083.926191]: cleared current plan +[INFO] [1736203083.927752]: moving to home +[INFO] [1736203087.437485]: reached home +[INFO] [1736203087.451589]: reset action history +[INFO] [1736203090.236288]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/requirements.txt b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-metadata.json b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c6b3d14e05c2732aa001f0e7219d6f782f1b722d --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-06T22:33:37.112692", + "startedAt": "2025-01-06T22:33:36.729865", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "22931875031f38fc79957f52ac8e49b1fb3d9fdc" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3936.9458750000003, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.97, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4495.522, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.979, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 346.4975051879883 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..3728f3254bc178a036c1997d5536ada6825f9834 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "5f6567d85218e9cc2a5076bccfe7c4a80a000044f84890f5a1722a66f128d7bb", "size": 1223582, "path": "media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4"}, "global_step": 664, "_timestamp": 1736203096.070415, "_runtime": 279.3310880661011, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "54887f1093ebbceb175ef9c830cac8037076d468a4f6bfc5ee4ab454b9f6e84c", "size": 1257490, "path": "media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4"}, "eval/num_success": 0.0, "episode": 10.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5290328860282898, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4204478859901428, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.33451345562934875, "eval/success_rate_lb_0.1": 0.009447986260056496, "eval/success_rate_ub_0.1": 0.20664875209331512, "eval/2_eval": {"_type": "video-file", "sha256": "5f1a4ab77045217b2016fb14d3fa82236d824a2bff91323047cd1e56d38a2ea0", "size": 1214564, "path": "media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "fd256e78c90500bf4c0d3d7238228f563d0f8bce065c57c00c528115140b02bf", "size": 1263661, "path": "media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "498f7f4f2ba24e669402c9dfec977c5b708a23374ae6ca9484423ee7deb31123", "size": 1224941, "path": "media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "36776135f97d857e6145c8894b46a34f285be837fe819b79f437deddabc06a55", "size": 350, "artifact_path": "wandb-client-artifact://t7derf2wrtdgepwjx2wseaflq1p3jzjexsvo0nk355281q5bgt719iw1ubj4p26nnhyirix7za9aemni3v2wyv0nqqchzvs3zbg1lqcggj692o3w6unehgvbv88i0csn/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://ux6d6fkkadk88tbl58f51ojq00kczs9tcnfiyldob1xugy0dx7dvamxwudz1w7fw516uxaxpx5qvfu1p8iyyuzha9m21fymyx57cadbxc1zhg4i1c6gbj65yh0244bwo:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 0, "total_num_episodes": 10, "total_success_rate": 0.0, "total_success_rate_lb_0.01": 0, "total_success_rate_ub_0.01": 0.3317304849961171, "total_success_rate_lb_0.025": 0, "total_success_rate_ub_0.025": 0.3041884303484581, "total_success_rate_lb_0.05": 0, "total_success_rate_ub_0.05": 0.1718106568516029, "total_success_rate_lb_0.1": 0, "total_success_rate_ub_0.1": 0.19463173306135229, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "99c5ea6d40a5d334ec51de757bf057a0ba78c69d3b5b7117fa337799eca330dd", "size": 437, "artifact_path": "wandb-client-artifact://50v8k7uaiur0p4r2hnu70ofzvifojextzkxkg48sd805vqe2vbrxwctf41qxkmo3pguvqshd4mkg6bs4d89o7jte2w0ftck8ow285jcoc24bugh0p962s6b62i0scjlx/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://a4fclib1ljji252azco4ww5oey49uq7f1hclqcofsjicok1jghgmhsss9kslnf7ui3h37ehkmi139fymzgccdtsmeh0mqkreqbehte0j8s625h92uknth43cvmb3o4sf:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 287}, "eval/total_success_rate": 0.0, "eval/total_success_rate_lb_0.01": 0.0, "eval/total_success_rate_ub_0.01": 0.3317304849624634, "eval/total_success_rate_lb_0.025": 0.0, "eval/total_success_rate_ub_0.025": 0.30418843030929565, "eval/total_success_rate_lb_0.05": 0.0, "eval/total_success_rate_ub_0.05": 0.17181065678596497, "eval/total_success_rate_lb_0.1": 0.0, "eval/total_success_rate_ub_0.1": 0.19463174045085907} \ No newline at end of file diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug-internal.log b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..d5d9fc363e7d6c1b5ce780068ffd220a9a21c5a2 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug-internal.log @@ -0,0 +1,804 @@ +2025-01-06 17:33:36,739 INFO StreamThr :2091683 [internal.py:wandb_internal():85] W&B internal server running at pid: 2091683, started at: 2025-01-06 17:33:36.739296 +2025-01-06 17:33:36,740 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status +2025-01-06 17:33:36,751 INFO WriterThread:2091683 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/run-i3pbiz9p.wandb +2025-01-06 17:33:36,751 DEBUG SenderThread:2091683 [sender.py:send():379] send: header +2025-01-06 17:33:36,760 DEBUG SenderThread:2091683 [sender.py:send():379] send: run +2025-01-06 17:33:37,035 INFO SenderThread:2091683 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files +2025-01-06 17:33:37,035 INFO SenderThread:2091683 [sender.py:_start_run_threads():1188] run started: i3pbiz9p with start time 1736202816.739327 +2025-01-06 17:33:37,051 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: check_version +2025-01-06 17:33:37,051 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: check_version +2025-01-06 17:33:37,094 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: run_start +2025-01-06 17:33:37,099 DEBUG HandlerThread:2091683 [system_info.py:__init__():26] System info init +2025-01-06 17:33:37,099 DEBUG HandlerThread:2091683 [system_info.py:__init__():41] System info init done +2025-01-06 17:33:37,099 INFO HandlerThread:2091683 [system_monitor.py:start():194] Starting system monitor +2025-01-06 17:33:37,099 INFO SystemMonitor:2091683 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-06 17:33:37,100 INFO HandlerThread:2091683 [system_monitor.py:probe():214] Collecting system info +2025-01-06 17:33:37,100 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started cpu monitoring +2025-01-06 17:33:37,101 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started disk monitoring +2025-01-06 17:33:37,101 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started gpu monitoring +2025-01-06 17:33:37,102 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started memory monitoring +2025-01-06 17:33:37,102 INFO SystemMonitor:2091683 [interfaces.py:start():188] Started network monitoring +2025-01-06 17:33:37,112 DEBUG HandlerThread:2091683 [system_info.py:probe():152] Probing system +2025-01-06 17:33:37,114 DEBUG HandlerThread:2091683 [system_info.py:_probe_git():137] Probing git +2025-01-06 17:33:37,118 DEBUG HandlerThread:2091683 [system_info.py:_probe_git():145] Probing git done +2025-01-06 17:33:37,119 DEBUG HandlerThread:2091683 [system_info.py:probe():200] Probing system done +2025-01-06 17:33:37,119 DEBUG HandlerThread:2091683 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-06T22:33:37.112692', 'startedAt': '2025-01-06T22:33:36.729865', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '22931875031f38fc79957f52ac8e49b1fb3d9fdc'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.9458750000003, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.97, 'min': 800.0, 'max': 4700.0}, {'current': 4495.522, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.979, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 346.4975051879883}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-06 17:33:37,119 INFO HandlerThread:2091683 [system_monitor.py:probe():224] Finished collecting system info +2025-01-06 17:33:37,119 INFO HandlerThread:2091683 [system_monitor.py:probe():227] Publishing system info +2025-01-06 17:33:37,119 DEBUG HandlerThread:2091683 [system_info.py:_save_code():44] Saving code +2025-01-06 17:33:37,120 DEBUG HandlerThread:2091683 [system_info.py:_save_code():67] Saving code done +2025-01-06 17:33:37,120 DEBUG HandlerThread:2091683 [system_info.py:_save_patches():84] Saving git patches +2025-01-06 17:33:37,182 DEBUG HandlerThread:2091683 [system_info.py:_save_patches():126] Saving git patches done +2025-01-06 17:33:37,182 INFO HandlerThread:2091683 [system_monitor.py:probe():229] Finished publishing system info +2025-01-06 17:33:37,185 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:33:37,185 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-06 17:33:37,186 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-06 17:33:37,186 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-06 17:33:37,429 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/d1l9j1pm-wandb-metadata.json +2025-01-06 17:33:37,435 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: python_packages +2025-01-06 17:33:37,435 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: python_packages +2025-01-06 17:33:37,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:33:37,436 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:33:37,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:37,481 INFO HandlerThread:2091683 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/tb" + save: true +} +control { + relay_id: "1924260d83af496aa73763b17d76d970" +} +_info { + stream_id: "i3pbiz9p" +} + +2025-01-06 17:33:37,481 DEBUG HandlerThread:2091683 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-06 17:33:37,482 INFO wandb-upload_1:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/02d26m1c-code/FISH/eval_robot.py +2025-01-06 17:33:37,499 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/3at6l5x8-diff.patch +2025-01-06 17:33:37,556 DEBUG SenderThread:2091683 [sender.py:send():379] send: telemetry +2025-01-06 17:33:37,556 DEBUG SenderThread:2091683 [sender.py:send():379] send: tbrecord +2025-01-06 17:33:37,557 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:33:37,557 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 with policy live +2025-01-06 17:33:37,751 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/360bta9l-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:33:38,037 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-metadata.json +2025-01-06 17:33:38,037 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/requirements.txt +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code/FISH/eval_robot.py +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/diff.patch +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code +2025-01-06 17:33:38,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code/FISH +2025-01-06 17:33:38,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:39,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:40,036 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:40,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:41,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:42,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:42,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:43,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:44,335 DEBUG SenderThread:2091683 [sender.py:send():379] send: config +2025-01-06 17:33:44,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:45,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:46,037 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:46,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:47,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:47,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:48,038 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:48,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:49,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:50,039 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:50,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:51,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:52,039 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:52,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:33:52,436 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:33:52,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:52,590 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:53,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:54,040 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:54,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:55,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:56,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:57,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:57,968 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:33:58,041 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:33:58,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:33:59,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:00,042 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:00,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:01,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:02,042 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:02,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:03,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:03,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:04,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:05,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:06,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:07,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:34:07,436 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:34:07,477 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:08,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:08,543 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:09,044 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml +2025-01-06 17:34:09,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:10,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:11,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:12,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:13,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:14,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:14,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:15,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:16,046 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:16,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:17,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:18,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:19,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:19,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:20,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:21,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:22,048 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:22,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:34:22,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:34:22,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:23,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:24,048 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:24,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:25,373 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:25,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:26,049 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:26,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:27,444 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:28,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:28,671 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:34:28,671 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 with policy now +2025-01-06 17:34:28,672 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:34:28,673 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:34:28,673 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:34:28,674 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:34:28,931 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/t0898jau-media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media +2025-01-06 17:34:29,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos +2025-01-06 17:34:29,337 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/49fouzfx-media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 +2025-01-06 17:34:29,444 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:30,050 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:30,444 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:30,678 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:31,444 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:32,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:33,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:34,051 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:34,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:35,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:36,053 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:36,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:36,637 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:37,103 DEBUG SystemMonitor:2091683 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-06 17:34:37,103 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:34:37,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:34:37,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:34:37,477 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:38,054 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:34:38,306 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/7gazhosy-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:34:38,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:39,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:40,055 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:40,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:41,446 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:42,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:42,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:43,446 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:44,056 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:34:44,447 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:45,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:46,447 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:47,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:47,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:48,447 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:49,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:50,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:51,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:52,436 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:34:52,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:34:52,477 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:52,588 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:53,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:54,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:55,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:56,448 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:57,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:58,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:34:58,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:34:59,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:00,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:01,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:02,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:03,439 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:03,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:04,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:05,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:06,449 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:07,111 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:35:07,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:35:07,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:35:07,477 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:08,450 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:08,968 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:09,371 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:35:09,371 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 with policy now +2025-01-06 17:35:09,371 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:35:09,372 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:35:09,372 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:35:09,372 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:35:09,450 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:09,628 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/v6xtb6yv-media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 +2025-01-06 17:35:10,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:35:10,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:10,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 +2025-01-06 17:35:10,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:35:10,324 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/lpqadr1r-media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 +2025-01-06 17:35:10,451 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:11,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:12,065 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:12,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:13,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:14,066 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:14,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:14,729 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:15,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:16,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:17,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:18,068 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:18,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:19,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:20,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:20,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:21,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:22,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:35:22,438 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:35:22,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:23,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:24,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:25,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:25,591 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:26,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:27,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:28,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:29,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:30,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:31,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:31,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:32,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:33,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:34,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:35,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:36,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:37,104 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:35:37,106 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:37,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:35:37,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:35:37,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:38,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:39,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:40,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:41,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:42,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:42,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:43,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:44,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:45,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:46,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:47,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:48,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:48,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:49,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:50,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:51,457 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:52,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:35:52,438 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:35:52,485 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:53,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:53,548 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:54,080 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:35:54,328 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/cywpf5cw-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:35:54,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:54,930 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:35:54,930 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:35:54,932 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:35:55,080 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:35:55,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:56,080 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:35:56,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:57,458 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:58,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:35:58,553 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:35:59,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:00,082 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:00,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:01,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:02,082 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:02,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:03,459 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:04,083 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:04,320 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:04,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:05,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:06,084 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:06,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:07,105 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:36:07,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:36:07,437 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:36:07,485 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:08,085 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:08,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:09,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:09,513 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:10,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:11,460 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:12,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:13,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:14,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:15,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:15,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:16,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:17,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:18,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:19,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:20,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:21,441 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:21,461 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:22,437 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:36:22,438 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:36:22,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:23,462 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:24,462 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:25,464 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:25,833 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:36:25,834 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:36:25,834 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 with policy now +2025-01-06 17:36:25,834 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:36:25,834 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:36:25,834 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:36:26,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:36:26,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:26,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 +2025-01-06 17:36:26,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:36:26,095 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/h1ny00v3-media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 +2025-01-06 17:36:26,359 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/0f2k6ehu-media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 +2025-01-06 17:36:26,468 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:26,835 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:27,466 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:28,091 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:28,468 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:29,466 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:30,092 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:30,469 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:31,465 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:32,022 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:32,093 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:32,464 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:33,472 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:34,093 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:34,467 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:34,747 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:36:34,747 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:36:34,748 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:36:35,094 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:36:35,094 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:36:35,466 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:36,094 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:36,468 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:37,105 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:36:37,105 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:37,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:36:37,441 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:36:37,481 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:38,095 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:38,507 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:39,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:40,096 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:40,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:41,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:42,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:42,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:43,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:44,499 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:45,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:46,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:47,443 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:47,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:48,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:49,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:50,500 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:51,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:52,100 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:52,438 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:36:52,438 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:36:52,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:52,518 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:53,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:54,100 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:54,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:55,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:56,101 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:56,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:57,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:57,549 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:36:58,102 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:36:58,501 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:36:59,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:00,103 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:00,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:01,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:02,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:03,446 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:03,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:04,502 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:05,533 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:06,505 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:06,849 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:37:06,850 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:37:06,850 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 with policy now +2025-01-06 17:37:06,851 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:37:06,851 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:37:06,851 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:37:07,103 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/qsdisp5l-media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 +2025-01-06 17:37:07,105 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:37:07,106 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 +2025-01-06 17:37:07,106 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:37:07,106 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:37:07,378 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/l92u06nl-media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 +2025-01-06 17:37:07,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:37:07,442 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:37:07,506 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:08,106 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:08,510 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:08,531 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:09,106 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:09,509 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:10,518 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:11,520 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:12,107 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:12,524 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:13,520 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:14,081 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:14,108 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:14,521 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:15,525 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:16,526 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:17,527 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:18,527 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:19,453 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:19,529 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:20,527 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:21,528 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:22,445 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:37:22,445 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:37:22,526 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:23,529 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:24,526 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:24,528 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:25,528 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:26,529 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:27,538 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:28,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:29,533 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:30,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:30,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:31,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:31,814 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:37:31,814 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:37:31,814 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:37:32,114 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:37:32,114 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:37:32,356 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/cjf8a6zg-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:37:32,533 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:33,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:34,115 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:34,537 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:35,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:35,561 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:36,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:37,106 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:37:37,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:37:37,440 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:37:37,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:38,116 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:37:38,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:39,534 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:40,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:40,562 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:41,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:42,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:43,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:44,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:45,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:46,455 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:46,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:47,535 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:48,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:49,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:50,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:51,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:52,440 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:37:52,441 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:37:52,536 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:52,591 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:53,537 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:54,537 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:55,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:56,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:57,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:58,454 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:37:58,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:37:59,546 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:00,548 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:01,548 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:02,124 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:02,567 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:03,559 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:03,724 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:03,902 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:38:03,902 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 with policy now +2025-01-06 17:38:03,905 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:38:03,905 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:03,905 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:03,911 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:04,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:04,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:04,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 +2025-01-06 17:38:04,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval +2025-01-06 17:38:04,164 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/t4rq8q6d-media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 +2025-01-06 17:38:04,429 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/mrycdmvc-media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 +2025-01-06 17:38:04,557 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:05,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:05,560 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:06,125 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:06,559 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:07,106 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:38:07,485 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:38:07,485 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:38:07,563 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:08,562 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:09,560 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:09,599 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:10,127 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:10,563 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:11,562 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:12,128 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:12,580 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:13,570 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:14,570 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:15,195 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:15,567 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:15,619 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:38:15,619 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:38:15,794 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:38:16,096 INFO wandb-upload_0:2091683 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpgtowj55l +2025-01-06 17:38:16,129 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:16,129 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:38:16,129 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:38:16,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media +2025-01-06 17:38:16,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table +2025-01-06 17:38:16,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval +2025-01-06 17:38:16,173 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-06 17:38:16,184 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: partial_history +2025-01-06 17:38:16,565 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:16,599 INFO SenderThread:2091683 [sender.py:send_request_log_artifact():1525] logged artifact run-i3pbiz9p-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk0Mzk0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMzMxNQ==', 'latestArtifact': None}} +2025-01-06 17:38:16,599 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:38:16,599 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-06 17:38:16,599 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:16,599 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,600 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,600 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:16,600 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,604 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,604 DEBUG SenderThread:2091683 [sender.py:send():379] send: telemetry +2025-01-06 17:38:16,604 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,604 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,605 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,605 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,605 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,606 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,606 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,606 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,606 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,607 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,607 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,607 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,607 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,608 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,608 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,608 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,608 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,609 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,609 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,609 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,609 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:16,610 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:16,610 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: log_artifact +2025-01-06 17:38:16,808 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/mhk63r2c-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:38:17,051 INFO wandb-upload_2:2091683 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpe9w99qyq +2025-01-06 17:38:17,105 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/zu2qupoz-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:38:17,129 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:17,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2025-01-06 17:38:17,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval +2025-01-06 17:38:17,427 INFO SenderThread:2091683 [sender.py:send_request_log_artifact():1525] logged artifact run-i3pbiz9p-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyMTk0Mzk2NA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMDUxMzMyMA==', 'latestArtifact': None}} +2025-01-06 17:38:17,427 DEBUG SenderThread:2091683 [sender.py:send():379] send: files +2025-01-06 17:38:17,428 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json with policy now +2025-01-06 17:38:17,428 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:17,428 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:17,428 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:17,565 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:17,611 INFO wandb-upload_1:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/ols7cqzq-media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2025-01-06 17:38:17,856 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/l5pgckog-media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2025-01-06 17:38:18,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:18,130 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:18,565 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:19,566 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:20,456 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:20,566 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:21,566 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:22,442 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: stop_status +2025-01-06 17:38:22,442 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: stop_status +2025-01-06 17:38:22,566 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:23,567 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:24,101 DEBUG SenderThread:2091683 [sender.py:send():379] send: exit +2025-01-06 17:38:24,101 INFO SenderThread:2091683 [sender.py:send_exit():586] handling exit code: 0 +2025-01-06 17:38:24,102 INFO SenderThread:2091683 [sender.py:send_exit():588] handling runtime: 287 +2025-01-06 17:38:24,104 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:24,104 INFO SenderThread:2091683 [sender.py:send_exit():594] send defer +2025-01-06 17:38:24,104 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,104 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-06 17:38:24,104 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:24,105 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-06 17:38:24,105 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 1 +2025-01-06 17:38:24,105 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,105 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-06 17:38:24,105 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:24,105 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-06 17:38:24,105 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 2 +2025-01-06 17:38:24,105 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,105 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-06 17:38:24,105 INFO HandlerThread:2091683 [system_monitor.py:finish():203] Stopping system monitor +2025-01-06 17:38:24,106 DEBUG SystemMonitor:2091683 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-06 17:38:24,106 DEBUG SystemMonitor:2091683 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-06 17:38:24,106 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined cpu monitor +2025-01-06 17:38:24,107 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined disk monitor +2025-01-06 17:38:24,133 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:24,160 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined gpu monitor +2025-01-06 17:38:24,160 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined memory monitor +2025-01-06 17:38:24,160 INFO HandlerThread:2091683 [interfaces.py:finish():200] Joined network monitor +2025-01-06 17:38:24,161 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:24,161 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-06 17:38:24,161 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 3 +2025-01-06 17:38:24,161 DEBUG SenderThread:2091683 [sender.py:send():379] send: stats +2025-01-06 17:38:24,161 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,161 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-06 17:38:24,161 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:24,161 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-06 17:38:24,161 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 4 +2025-01-06 17:38:24,161 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:24,161 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-06 17:38:26,133 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:30,872 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:30,872 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:30,872 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: keepalive +2025-01-06 17:38:30,873 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:30,873 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-06 17:38:30,873 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 5 +2025-01-06 17:38:30,873 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:30,873 DEBUG SenderThread:2091683 [sender.py:send():379] send: history +2025-01-06 17:38:30,873 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: summary_record +2025-01-06 17:38:30,873 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:30,874 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-06 17:38:30,874 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:30,874 DEBUG SenderThread:2091683 [sender.py:send():379] send: summary +2025-01-06 17:38:30,878 INFO SenderThread:2091683 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-06 17:38:30,878 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:30,878 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-06 17:38:30,878 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 6 +2025-01-06 17:38:30,878 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:30,879 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-06 17:38:30,879 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:30,879 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-06 17:38:30,882 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: status_report +2025-01-06 17:38:31,003 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 7 +2025-01-06 17:38:31,003 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:31,003 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-06 17:38:31,004 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:31,004 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-06 17:38:31,103 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:31,135 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml +2025-01-06 17:38:31,136 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:32,931 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 8 +2025-01-06 17:38:32,931 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:32,932 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:32,932 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-06 17:38:32,932 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:32,932 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-06 17:38:32,932 INFO SenderThread:2091683 [job_builder.py:build():440] Attempting to build job artifact +2025-01-06 17:38:32,935 INFO SenderThread:2091683 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-06 17:38:32,937 INFO SenderThread:2091683 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-06 17:38:32,958 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 9 +2025-01-06 17:38:32,958 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:32,958 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-06 17:38:32,959 DEBUG SenderThread:2091683 [sender.py:send():379] send: artifact +2025-01-06 17:38:33,103 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:33,136 INFO Thread-12 :2091683 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:33,188 INFO SenderThread:2091683 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyMTkzNjUwMA==', 'versionIndex': 424}}} +2025-01-06 17:38:33,188 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:33,188 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-06 17:38:33,188 INFO SenderThread:2091683 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-06 17:38:34,137 INFO SenderThread:2091683 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files +2025-01-06 17:38:34,137 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml config.yaml +2025-01-06 17:38:34,137 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json wandb-summary.json +2025-01-06 17:38:34,137 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:38:34,138 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/requirements.txt requirements.txt +2025-01-06 17:38:34,138 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-metadata.json wandb-metadata.json +2025-01-06 17:38:34,138 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log output.log +2025-01-06 17:38:34,138 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/diff.patch diff.patch +2025-01-06 17:38:34,141 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-06 17:38:34,141 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 media/videos/eval/3_eval_3_fd256e78c90500bf4c0d.mp4 +2025-01-06 17:38:34,142 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 media/videos/eval/0_eval_0_5f6567d85218e9cc2a50.mp4 +2025-01-06 17:38:34,142 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 media/videos/eval/1_eval_1_54887f1093ebbceb175e.mp4 +2025-01-06 17:38:34,143 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 media/videos/eval/2_eval_2_5f1a4ab77045217b2016.mp4 +2025-01-06 17:38:34,143 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 media/videos/eval/4_eval_4_498f7f4f2ba24e669402.mp4 +2025-01-06 17:38:34,146 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json media/table/eval/total_success_rate_ci_6_99c5ea6d40a5d334ec51.table.json +2025-01-06 17:38:34,146 INFO SenderThread:2091683 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-06 17:38:34,146 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 10 +2025-01-06 17:38:34,146 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:34,146 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,146 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-06 17:38:34,147 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,147 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-06 17:38:34,147 INFO SenderThread:2091683 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:38:34,316 INFO wandb-upload_2:2091683 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/wandb-summary.json +2025-01-06 17:38:34,332 INFO wandb-upload_1:2091683 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/config.yaml +2025-01-06 17:38:34,398 INFO wandb-upload_0:2091683 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/output.log +2025-01-06 17:38:34,420 INFO wandb-upload_3:2091683 [upload_job.py:push():130] Uploaded file /tmp/tmpkwgox9q9wandb/urq8oc1x-events.out.tfevents.1736202817.leonmkim-ROG-Strix-G15CS-G15CS.2091517.0 +2025-01-06 17:38:34,423 INFO wandb-upload_4:2091683 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/files/requirements.txt +2025-01-06 17:38:34,623 INFO Thread-11 (_thread_body):2091683 [sender.py:transition_state():614] send defer: 11 +2025-01-06 17:38:34,623 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,623 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-06 17:38:34,624 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,624 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-06 17:38:34,624 INFO SenderThread:2091683 [file_pusher.py:join():175] waiting for file pusher +2025-01-06 17:38:34,625 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 12 +2025-01-06 17:38:34,625 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,625 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-06 17:38:34,625 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,625 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-06 17:38:34,625 INFO SenderThread:2091683 [file_stream.py:finish():601] file stream finish called +2025-01-06 17:38:34,677 INFO SenderThread:2091683 [file_stream.py:finish():605] file stream finish is done +2025-01-06 17:38:34,677 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 13 +2025-01-06 17:38:34,677 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,677 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-06 17:38:34,678 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,678 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-06 17:38:34,678 INFO SenderThread:2091683 [sender.py:transition_state():614] send defer: 14 +2025-01-06 17:38:34,678 DEBUG SenderThread:2091683 [sender.py:send():379] send: final +2025-01-06 17:38:34,678 DEBUG SenderThread:2091683 [sender.py:send():379] send: footer +2025-01-06 17:38:34,678 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: defer +2025-01-06 17:38:34,678 INFO HandlerThread:2091683 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-06 17:38:34,678 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: defer +2025-01-06 17:38:34,678 INFO SenderThread:2091683 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-06 17:38:34,679 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:34,679 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:34,679 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: server_info +2025-01-06 17:38:34,680 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-06 17:38:34,680 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: server_info +2025-01-06 17:38:34,681 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: get_summary +2025-01-06 17:38:34,682 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-06 17:38:34,682 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-06 17:38:34,743 DEBUG SenderThread:2091683 [sender.py:send_request():406] send_request: poll_exit +2025-01-06 17:38:34,744 INFO MainThread:2091683 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-06 17:38:34,744 INFO MainThread:2091683 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-06 17:38:34,744 INFO MainThread:2091683 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-06 17:38:34,744 DEBUG HandlerThread:2091683 [handler.py:handle_request():158] handle_request: shutdown +2025-01-06 17:38:34,744 INFO HandlerThread:2091683 [handler.py:finish():882] shutting down handler +2025-01-06 17:38:35,680 INFO WriterThread:2091683 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/run-i3pbiz9p.wandb +2025-01-06 17:38:35,743 INFO SenderThread:2091683 [sender.py:finish():1615] shutting down sender +2025-01-06 17:38:35,744 INFO SenderThread:2091683 [file_pusher.py:finish():169] shutting down file pusher +2025-01-06 17:38:35,744 INFO SenderThread:2091683 [file_pusher.py:join():175] waiting for file pusher diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug.log b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9bc20b05ab5ea3ad1f6e5ebe7d2744b094bc4291 --- /dev/null +++ b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Configure stats pid to 2091517 +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/settings +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug.log +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/logs/debug-internal.log +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:init():569] calling init triggers +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p'} +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:init():619] starting backend +2025-01-06 17:33:36,731 INFO MainThread:2091517 [wandb_init.py:init():623] setting up manager +2025-01-06 17:33:36,738 INFO MainThread:2091517 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-06 17:33:36,739 INFO MainThread:2091517 [wandb_init.py:init():631] backend started and connected +2025-01-06 17:33:36,750 INFO MainThread:2091517 [wandb_init.py:init():720] updated telemetry +2025-01-06 17:33:36,758 INFO MainThread:2091517 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-06 17:33:37,050 INFO MainThread:2091517 [wandb_run.py:_on_init():2435] communicating current version +2025-01-06 17:33:37,090 INFO MainThread:2091517 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.1 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-06 17:33:37,090 INFO MainThread:2091517 [wandb_init.py:init():804] starting run threads in backend +2025-01-06 17:33:37,436 INFO MainThread:2091517 [wandb_run.py:_console_start():2413] atexit reg +2025-01-06 17:33:37,436 INFO MainThread:2091517 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-06 17:33:37,437 INFO MainThread:2091517 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-06 17:33:37,437 INFO MainThread:2091517 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-06 17:33:37,440 INFO MainThread:2091517 [wandb_init.py:init():847] run started, returning control to user process +2025-01-06 17:33:37,441 INFO MainThread:2091517 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/i3pbiz9p/tb, True +2025-01-06 17:33:44,335 INFO MainThread:2091517 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'greece', 'left_book_slot': 'twodim'} +2025-01-06 17:38:35,745 WARNING MsgRouterThr:2091517 [router.py:message_loop():77] message_loop has been closed diff --git a/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/run-i3pbiz9p.wandb b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/run-i3pbiz9p.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b57a47dd1f857e93d9c7532a555dbbc747d00f75 Binary files /dev/null and b/i3pbiz9p/wandb/run-20250106_173336-i3pbiz9p/run-i3pbiz9p.wandb differ diff --git a/kq00a7xd/.hydra/config.yaml b/kq00a7xd/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e992f03a53006e05f336e80f0efc3330a2c0973 --- /dev/null +++ b/kq00a7xd/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1022_0' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/kq00a7xd/.hydra/hydra.yaml b/kq00a7xd/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80929239f693631c817ba066a5e3a0e030006ef5 --- /dev/null +++ b/kq00a7xd/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/kq00a7xd/.hydra/overrides.yaml b/kq00a7xd/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/kq00a7xd/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/kq00a7xd/episode_rosbags/aligned_depth_to_color_K.npy b/kq00a7xd/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kq00a7xd/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kq00a7xd/episode_rosbags/cam_tf_world.npy b/kq00a7xd/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/kq00a7xd/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/kq00a7xd/episode_rosbags/color_K.npy b/kq00a7xd/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/kq00a7xd/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/kq00a7xd/episode_rosbags/depth_K.npy b/kq00a7xd/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/kq00a7xd/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/kq00a7xd/episode_rosbags/episode_0_2025-01-07-18-01-28.bag b/kq00a7xd/episode_rosbags/episode_0_2025-01-07-18-01-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..a31f6a21f0993e0844477d49051f627f76c515b9 --- /dev/null +++ b/kq00a7xd/episode_rosbags/episode_0_2025-01-07-18-01-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea296e4457c6fe12b314d0499ff62e664eedea38b495acd919560c6d9dcdfdd7 +size 1409089539 diff --git a/kq00a7xd/episode_rosbags/episode_1_2025-01-07-18-02-10.bag b/kq00a7xd/episode_rosbags/episode_1_2025-01-07-18-02-10.bag new file mode 100644 index 0000000000000000000000000000000000000000..617641644af7ac712c6ee8268dd52790549d6e4b --- /dev/null +++ b/kq00a7xd/episode_rosbags/episode_1_2025-01-07-18-02-10.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:535018dee6f81904ade6d721460f77078c6fcf77dd16e7efa61a03d693a6b946 +size 1413497317 diff --git a/kq00a7xd/episode_rosbags/episode_4_2025-01-07-18-04-36.bag b/kq00a7xd/episode_rosbags/episode_4_2025-01-07-18-04-36.bag new file mode 100644 index 0000000000000000000000000000000000000000..2d39686df580b6e1c622c7690e9ccfae99446d36 --- /dev/null +++ b/kq00a7xd/episode_rosbags/episode_4_2025-01-07-18-04-36.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0500c743b8cb5522de342df0d50568feb911cac9a02567a40191574a06d389a9 +size 1410857789 diff --git a/kq00a7xd/eval_robot.log b/kq00a7xd/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..bdb0aa5fa3fcd6e97c3c796abcc56107663f319a --- /dev/null +++ b/kq00a7xd/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 18:00:34,843][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 18:00:34,846][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/kq00a7xd/eval_video/0_eval.mp4 b/kq00a7xd/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d9eaefcbd93b6a5a58c1cf06406e4b66189f24f4 --- /dev/null +++ b/kq00a7xd/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79f998250d76be90139304bc0b220414701797d2b24d46ca25335d7103e0f0eb +size 1134868 diff --git a/kq00a7xd/eval_video/1_eval.mp4 b/kq00a7xd/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..263e43255d721a4044e74bb19f6b0b7fdf1474b8 --- /dev/null +++ b/kq00a7xd/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5777bf060ba6d7d77b457c7a7c8baf649e3ceefda782c804d6dae6ce969d0b1b +size 1129184 diff --git a/kq00a7xd/eval_video/2_eval.mp4 b/kq00a7xd/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..611fc00bab9f6423719e09337a2aa83cc293dd9b --- /dev/null +++ b/kq00a7xd/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a16b4848a19dfb39015158b10baf969d1b0db6dde103aec772d93c954a320f32 +size 1138006 diff --git a/kq00a7xd/eval_video/3_eval.mp4 b/kq00a7xd/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73bb8ff6b4f4b107be975911ce0be89b030a17f4 --- /dev/null +++ b/kq00a7xd/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:796eb7d92e1e7dcc5b92d8e7694013ab0e0cf7772c2da0a730543632ca0a7bd6 +size 1133100 diff --git a/kq00a7xd/eval_video/4_eval.mp4 b/kq00a7xd/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cebb051b7f8e8d6b47a6793e18e426b75b4d5279 --- /dev/null +++ b/kq00a7xd/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:227b80e6fd7f10383222c5cf88ae51a081022c5826eab44672ea2b7b2d6fadbb +size 1142083 diff --git a/kq00a7xd/tb/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 b/kq00a7xd/tb/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 new file mode 100644 index 0000000000000000000000000000000000000000..38cf14af0665a96cb95c4696b286e2c9d13f1b2f --- /dev/null +++ b/kq00a7xd/tb/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f3d05ad0147edc43e30e17f944c31564f65083e41744e151732702ef54254925 +size 6539 diff --git a/kq00a7xd/wandb/debug-internal.log b/kq00a7xd/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5dfd1a1d795757374af03ce2b011908a2c084235 --- /dev/null +++ b/kq00a7xd/wandb/debug-internal.log @@ -0,0 +1,810 @@ +2025-01-07 18:00:40,784 INFO StreamThr :2206495 [internal.py:wandb_internal():85] W&B internal server running at pid: 2206495, started at: 2025-01-07 18:00:40.784395 +2025-01-07 18:00:40,787 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:00:40,799 INFO WriterThread:2206495 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/run-kq00a7xd.wandb +2025-01-07 18:00:40,799 DEBUG SenderThread:2206495 [sender.py:send():379] send: header +2025-01-07 18:00:40,806 DEBUG SenderThread:2206495 [sender.py:send():379] send: run +2025-01-07 18:00:41,054 INFO SenderThread:2206495 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files +2025-01-07 18:00:41,054 INFO SenderThread:2206495 [sender.py:_start_run_threads():1188] run started: kq00a7xd with start time 1736290840.784371 +2025-01-07 18:00:41,068 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:00:41,069 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:00:41,113 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:00:41,118 DEBUG HandlerThread:2206495 [system_info.py:__init__():26] System info init +2025-01-07 18:00:41,118 DEBUG HandlerThread:2206495 [system_info.py:__init__():41] System info init done +2025-01-07 18:00:41,118 INFO HandlerThread:2206495 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:00:41,118 INFO SystemMonitor:2206495 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:00:41,118 INFO HandlerThread:2206495 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:00:41,119 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:00:41,120 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:00:41,120 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:00:41,121 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:00:41,121 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:00:41,131 DEBUG HandlerThread:2206495 [system_info.py:probe():152] Probing system +2025-01-07 18:00:41,132 DEBUG HandlerThread:2206495 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:00:41,137 DEBUG HandlerThread:2206495 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:00:41,137 DEBUG HandlerThread:2206495 [system_info.py:probe():200] Probing system done +2025-01-07 18:00:41,137 DEBUG HandlerThread:2206495 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:00:41.131479', 'startedAt': '2025-01-07T23:00:40.774898', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4122.291499999999, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.584, 'min': 800.0, 'max': 4700.0}, {'current': 4497.364, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 796.296257019043}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:00:41,137 INFO HandlerThread:2206495 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:00:41,137 INFO HandlerThread:2206495 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:00:41,137 DEBUG HandlerThread:2206495 [system_info.py:_save_code():44] Saving code +2025-01-07 18:00:41,139 DEBUG HandlerThread:2206495 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:00:41,139 DEBUG HandlerThread:2206495 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:00:41,200 DEBUG HandlerThread:2206495 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:00:41,201 INFO HandlerThread:2206495 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:00:41,204 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:00:41,204 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:00:41,204 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:00:41,204 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 18:00:41,433 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:00:41,433 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:00:41,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:41,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:00:41,435 DEBUG SenderThread:2206495 [sender.py:send():379] send: telemetry +2025-01-07 18:00:41,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:00:41,443 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/0c0rsn34-wandb-metadata.json +2025-01-07 18:00:41,469 INFO wandb-upload_2:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/uxde6usb-diff.patch +2025-01-07 18:00:41,477 INFO HandlerThread:2206495 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/tb" + save: true +} +control { + relay_id: "b4011873ceab42ec84f50f4786cd5445" +} +_info { + stream_id: "kq00a7xd" +} + +2025-01-07 18:00:41,477 DEBUG HandlerThread:2206495 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:00:41,524 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/23yb9kz3-code/FISH/eval_robot.py +2025-01-07 18:00:41,552 DEBUG SenderThread:2206495 [sender.py:send():379] send: tbrecord +2025-01-07 18:00:41,553 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:00:41,553 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 with policy live +2025-01-07 18:00:41,754 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/uyy54zab-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-metadata.json +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/diff.patch +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code/FISH/eval_robot.py +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/requirements.txt +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code/FISH +2025-01-07 18:00:42,057 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code +2025-01-07 18:00:42,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:43,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:44,055 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:44,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:45,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:46,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:46,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:47,008 DEBUG SenderThread:2206495 [sender.py:send():379] send: config +2025-01-07 18:00:47,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:48,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:48,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:49,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:50,057 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:50,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:51,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:51,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:52,057 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:52,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:53,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:54,058 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:54,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:55,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:56,059 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:56,433 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:00:56,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:00:56,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:56,552 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:57,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:58,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:59,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:00,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:01,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:02,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:02,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:03,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:04,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:05,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:06,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:07,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:07,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:08,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:09,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:10,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:11,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:01:11,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:01:11,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:12,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:12,573 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:13,063 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml +2025-01-07 18:01:13,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:14,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:15,439 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:16,439 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:17,439 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:18,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:18,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:19,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:20,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:21,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:22,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:23,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:24,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:24,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:25,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:26,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:01:26,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:01:26,441 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:27,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:28,067 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:28,441 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:29,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:30,068 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:30,122 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:30,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:31,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:32,069 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:32,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:33,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:34,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:35,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:35,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:36,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:37,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:38,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:39,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:40,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:41,121 DEBUG SystemMonitor:2206495 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:01:41,122 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:01:41,122 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:41,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:01:41,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:01:41,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:42,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:43,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:44,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:45,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:46,072 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:46,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:46,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:47,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:48,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:49,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:50,074 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:50,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:51,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:51,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:52,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:53,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:54,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:55,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:56,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:01:56,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:01:56,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:56,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:56,767 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:01:56,767 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_79f998250d76be901393.mp4 with policy now +2025-01-07 18:01:56,767 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:01:56,768 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:01:56,768 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:01:56,769 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:01:57,030 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/4bd47ty3-media/videos/eval/0_eval_0_79f998250d76be901393.mp4 +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/0_eval_0_79f998250d76be901393.mp4 +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:01:57,353 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/n69nlqr1-media/videos/eval/0_eval_0_79f998250d76be901393.mp4 +2025-01-07 18:01:57,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:58,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:58,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:59,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:00,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:01,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:02,077 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:02,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:02,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:03,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:04,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:05,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:06,079 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:06,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:07,079 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:02:07,347 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/tjryokvc-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:02:07,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:07,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:08,079 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:08,448 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:09,448 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:10,080 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:10,451 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:11,122 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:02:11,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:02:11,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:02:11,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:12,081 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:12,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:12,500 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:13,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:14,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:15,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:16,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:17,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:18,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:18,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:19,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:20,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:21,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:22,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:23,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:23,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:24,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:25,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:26,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:02:26,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:02:26,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:27,455 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:28,455 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:28,551 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:29,456 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:30,456 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:31,456 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:32,457 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:33,457 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:34,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:34,459 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:35,462 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:36,459 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:37,459 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:38,089 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:38,456 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:02:38,456 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 with policy now +2025-01-07 18:02:38,459 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:02:38,460 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:02:38,460 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:02:38,460 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:02:38,461 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:38,724 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/ofn66gmw-media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 +2025-01-07 18:02:39,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:02:39,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 +2025-01-07 18:02:39,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:02:39,378 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/6vngklpe-media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 +2025-01-07 18:02:39,460 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:39,510 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:40,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:40,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:41,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:41,123 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:02:41,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:02:41,443 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:02:41,489 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:42,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:43,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:44,091 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:44,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:44,988 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:45,465 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:46,092 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:46,465 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:47,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:48,465 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:49,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:50,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:50,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:51,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:52,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:53,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:54,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:55,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:55,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:56,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:02:56,436 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:02:56,481 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:57,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:58,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:59,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:00,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:00,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:01,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:02,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:03,098 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:03:03,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:03,924 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:03:03,924 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:03:03,924 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:03:04,098 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:03:04,098 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:04,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:05,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:05,925 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:06,099 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:06,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:07,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:08,100 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:08,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:09,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:10,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:10,994 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:11,123 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:03:11,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:03:11,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:03:11,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:12,479 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:13,482 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:14,482 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:15,482 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:16,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:16,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:17,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:18,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:19,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:20,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:21,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:21,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:22,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:23,485 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:24,485 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:25,485 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:26,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:03:26,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:03:26,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:26,561 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:27,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:28,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:29,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:30,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:31,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:32,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:32,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:33,487 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:34,490 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:34,771 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:03:34,771 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 with policy now +2025-01-07 18:03:34,774 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:03:34,774 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:03:34,774 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:03:34,775 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:03:35,044 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/1lad65yv-media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 +2025-01-07 18:03:35,109 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:03:35,109 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 +2025-01-07 18:03:35,109 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:03:35,403 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/jj1jq58q-media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 +2025-01-07 18:03:35,490 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:36,109 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:36,488 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:37,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:37,500 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:38,502 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:39,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:40,110 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:40,514 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:41,124 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:03:41,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:03:41,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:03:41,501 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:42,492 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:42,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:43,501 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:43,752 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:03:43,752 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:03:43,752 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:03:44,112 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:03:44,112 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:44,112 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:03:44,384 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/lqehp4vu-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:03:44,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:45,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:46,113 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:46,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:47,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:48,114 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:48,281 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:48,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:49,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:50,114 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:50,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:51,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:52,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:53,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:53,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:54,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:55,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:56,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:03:56,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:03:56,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:57,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:58,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:58,531 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:59,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:00,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:01,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:02,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:03,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:04,119 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:04,257 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:04,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:05,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:06,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:07,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:08,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:09,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:09,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:10,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:11,124 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:04:11,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:04:11,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:04:11,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:12,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:13,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:14,513 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:14,793 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:15,089 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:04:15,089 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 with policy now +2025-01-07 18:04:15,092 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:04:15,092 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:04:15,092 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:04:15,092 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:04:15,123 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:04:15,123 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 +2025-01-07 18:04:15,123 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:04:15,366 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/b3sruz0y-media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 +2025-01-07 18:04:15,510 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:15,641 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/fuqlh64b-media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 +2025-01-07 18:04:16,123 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:16,510 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:17,516 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:18,510 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:19,513 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:19,869 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:20,125 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:20,509 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:21,517 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:22,519 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:23,519 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:24,126 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:24,516 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:25,517 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:25,662 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:26,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:04:26,446 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:04:26,520 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:27,520 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:28,523 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:29,522 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:30,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:31,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:31,521 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:32,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:32,811 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:04:32,811 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:04:32,811 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:04:33,129 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:04:33,129 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:04:33,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:34,129 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:34,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:35,527 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:36,130 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:36,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:36,527 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:37,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:38,131 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:38,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:39,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:40,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:41,125 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:04:41,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:04:41,437 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:04:41,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:41,591 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:42,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:43,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:44,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:45,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:46,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:47,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:47,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:48,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:49,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:50,135 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:50,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:51,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:52,135 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:52,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:52,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:53,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:54,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:55,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:56,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:04:56,437 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:04:56,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:57,498 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:57,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:58,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:59,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:00,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:01,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:02,527 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:03,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:03,527 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:04,539 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:04,970 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:05:04,971 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 with policy now +2025-01-07 18:05:04,973 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:05:04,974 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:04,974 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:04,974 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:05,140 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:05,140 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 +2025-01-07 18:05:05,140 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:05:05,247 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/dfrn8fuh-media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 +2025-01-07 18:05:05,529 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/etj2slca-media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 +2025-01-07 18:05:05,538 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:06,140 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:06,534 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:07,536 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:08,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:08,543 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:09,536 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:10,142 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:10,537 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:11,125 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:05:11,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:05:11,439 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:05:11,540 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:12,541 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:13,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:13,601 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:14,143 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:14,556 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:15,552 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:16,559 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:17,548 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:18,548 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:19,133 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:19,145 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:19,428 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/9wvzz7mh-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:19,544 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:19,868 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:19,869 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:19,869 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,090 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:05:20,090 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:05:20,145 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:20,145 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:20,145 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:20,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:05:20,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media +2025-01-07 18:05:20,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval +2025-01-07 18:05:20,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table +2025-01-07 18:05:20,232 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:05:20,342 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/f853rohi-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:05:20,548 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:20,569 INFO wandb-upload_1:2206495 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplfox_w0_ +2025-01-07 18:05:20,654 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:05:20,662 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:05:20,942 INFO SenderThread:2206495 [sender.py:send_request_log_artifact():1525] logged artifact run-kq00a7xd-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjA0ODY3Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTczNzE5OQ==', 'latestArtifact': None}} +2025-01-07 18:05:20,942 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:05:20,942 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 18:05:20,942 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:20,942 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,943 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,943 DEBUG SenderThread:2206495 [sender.py:send():379] send: telemetry +2025-01-07 18:05:20,943 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,947 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,947 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,947 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,948 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,948 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,948 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,948 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,949 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,949 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,949 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,949 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,950 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,950 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,950 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,950 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,951 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,951 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,951 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,951 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,952 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,952 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,952 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:05:21,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:21,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:21,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 18:05:21,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval +2025-01-07 18:05:21,211 INFO wandb-upload_2:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/1gg6gwtw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:05:21,323 INFO wandb-upload_1:2206495 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6ju784l7 +2025-01-07 18:05:21,357 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/k5py8w0s-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 18:05:21,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:21,725 INFO SenderThread:2206495 [sender.py:send_request_log_artifact():1525] logged artifact run-kq00a7xd-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjA0ODc3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTczNzI3OA==', 'latestArtifact': None}} +2025-01-07 18:05:21,726 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:05:21,726 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json with policy now +2025-01-07 18:05:21,726 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:21,726 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:21,727 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:21,978 INFO wandb-upload_2:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/9gsncjd5-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 18:05:22,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:22,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:22,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:23,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:24,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:24,728 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:25,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:26,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:05:26,437 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:05:26,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:27,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:28,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:29,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:30,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:30,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:31,150 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml +2025-01-07 18:05:31,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:32,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:33,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:34,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:35,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:36,457 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:36,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:37,547 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:38,581 DEBUG SenderThread:2206495 [sender.py:send():379] send: exit +2025-01-07 18:05:38,582 INFO SenderThread:2206495 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:05:38,582 INFO SenderThread:2206495 [sender.py:send_exit():588] handling runtime: 297 +2025-01-07 18:05:38,582 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:38,583 INFO SenderThread:2206495 [sender.py:send_exit():594] send defer +2025-01-07 18:05:38,583 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,583 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:05:38,583 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:38,583 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:05:38,583 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:05:38,583 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,583 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:05:38,583 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:38,583 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:05:38,584 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:05:38,584 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,584 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:05:38,584 INFO HandlerThread:2206495 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:05:38,584 DEBUG SystemMonitor:2206495 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:05:38,584 DEBUG SystemMonitor:2206495 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:05:38,584 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:05:38,585 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:05:38,654 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:05:38,655 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:05:38,655 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:05:38,655 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:38,655 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:05:38,655 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:05:38,655 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:05:38,656 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,656 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:05:38,656 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:38,656 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:05:38,656 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:05:38,656 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,656 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:05:39,153 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:40,153 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:44,876 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:44,876 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:44,876 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:05:44,877 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:44,877 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:05:44,878 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:05:44,878 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:44,878 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:44,878 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:44,878 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:05:44,878 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:44,879 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:44,879 DEBUG SenderThread:2206495 [sender.py:send():379] send: summary +2025-01-07 18:05:44,879 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:44,880 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:05:44,880 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:44,880 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:05:44,880 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:05:44,880 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:44,880 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:44,880 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:05:44,880 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:05:45,155 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:45,583 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:46,623 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:05:46,623 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:46,624 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:46,624 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:05:46,624 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:46,624 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:05:46,624 INFO SenderThread:2206495 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:05:46,626 INFO SenderThread:2206495 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:05:46,628 INFO SenderThread:2206495 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:05:46,642 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:05:46,642 DEBUG SenderThread:2206495 [sender.py:send():379] send: artifact +2025-01-07 18:05:46,642 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:46,645 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:05:46,829 INFO SenderThread:2206495 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:05:46,829 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:46,829 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:05:46,829 INFO SenderThread:2206495 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml config.yaml +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json wandb-summary.json +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/requirements.txt requirements.txt +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:05:47,158 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log output.log +2025-01-07 18:05:47,158 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/diff.patch diff.patch +2025-01-07 18:05:47,158 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/0_eval_0_79f998250d76be901393.mp4 media/videos/eval/0_eval_0_79f998250d76be901393.mp4 +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 +2025-01-07 18:05:47,166 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 18:05:47,166 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:05:47,166 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:05:47,167 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,167 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:05:47,167 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,167 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:05:47,167 INFO SenderThread:2206495 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:05:47,339 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml +2025-01-07 18:05:47,391 INFO wandb-upload_2:2206495 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:47,391 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:47,392 INFO wandb-upload_4:2206495 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/requirements.txt +2025-01-07 18:05:47,434 INFO wandb-upload_3:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/375vrvdc-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:47,584 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:47,584 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:47,634 INFO Thread-11 (_thread_body):2206495 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:05:47,635 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,635 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:05:47,635 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,635 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:05:47,635 INFO SenderThread:2206495 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:05:47,636 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:05:47,637 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,637 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:05:47,637 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,637 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:05:47,637 INFO SenderThread:2206495 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:05:47,849 INFO SenderThread:2206495 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:05:47,849 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:05:47,849 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,849 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:05:47,849 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,849 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:05:47,849 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:05:47,849 DEBUG SenderThread:2206495 [sender.py:send():379] send: final +2025-01-07 18:05:47,849 DEBUG SenderThread:2206495 [sender.py:send():379] send: footer +2025-01-07 18:05:47,849 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,850 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:05:47,850 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,850 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:05:47,850 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:47,850 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:05:47,850 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:47,851 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:47,851 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:05:47,852 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:05:47,853 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:05:47,854 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:47,904 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:47,904 INFO MainThread:2206495 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:05:47,905 INFO MainThread:2206495 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:05:47,906 INFO MainThread:2206495 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:05:47,906 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:05:47,906 INFO HandlerThread:2206495 [handler.py:finish():882] shutting down handler +2025-01-07 18:05:48,851 INFO WriterThread:2206495 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/run-kq00a7xd.wandb +2025-01-07 18:05:48,904 INFO SenderThread:2206495 [sender.py:finish():1615] shutting down sender +2025-01-07 18:05:48,904 INFO SenderThread:2206495 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:05:48,904 INFO SenderThread:2206495 [file_pusher.py:join():175] waiting for file pusher diff --git a/kq00a7xd/wandb/debug.log b/kq00a7xd/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..aaf2bf19add5c913aed1e16b2238cb40095bf93f --- /dev/null +++ b/kq00a7xd/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Configure stats pid to 2206328 +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/settings +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:00:40,777 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug.log +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug-internal.log +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd'} +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:init():619] starting backend +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:init():623] setting up manager +2025-01-07 18:00:40,783 INFO MainThread:2206328 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:00:40,784 INFO MainThread:2206328 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:00:40,798 INFO MainThread:2206328 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:00:40,804 INFO MainThread:2206328 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:00:41,068 INFO MainThread:2206328 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:00:41,107 INFO MainThread:2206328 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:00:41,107 INFO MainThread:2206328 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:00:41,434 INFO MainThread:2206328 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:00:41,434 INFO MainThread:2206328 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:00:41,434 INFO MainThread:2206328 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:00:41,434 INFO MainThread:2206328 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:00:41,435 INFO MainThread:2206328 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:00:41,436 INFO MainThread:2206328 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/tb, True +2025-01-07 18:00:47,007 INFO MainThread:2206328 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:05:48,907 WARNING MsgRouterThr:2206328 [router.py:message_loop():77] message_loop has been closed diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code/FISH/eval_robot.py b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a8aaac7071b3e3478237f3978b6923d212536b09 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 0 +dataset_shuffle_seed: + desc: null + value: 1 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1022_0' +wandb_notes: + desc: null + value: 1022_0_req_1348_0restarted_2 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: false +use_sdf_maps: + desc: null + value: false +use_normals_maps: + desc: null + value: false +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 0 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: false + use_sdf_maps: false + use_normals_maps: false + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736290840 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/diff.patch b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..1a994978ad1dd8e2d18bdda54a765e73f2c0b5be --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/diff.patch @@ -0,0 +1,19 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..bc04fba 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -135,12 +135,12 @@ load_checkpoint: ${agent.load_checkpoint} + # RGBD+mask+act history + # wandb_run_id: '1002_0' + # wandb_run_id: '1009_1' # seed 1 +-# wandb_run_id: '1022_0' # dataset shuffle seed 1 ++wandb_run_id: '1022_0' # dataset shuffle seed 1 + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' + # wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json new file mode 100644 index 0000000000000000000000000000000000000000..abffe612675a3f5ba7b32314013cd92e261fa3b0 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.449680686007073, 1, 0.47999376058978616, 1, 0.6638181209236573, 1, 1, 1]]} \ No newline at end of file diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json new file mode 100644 index 0000000000000000000000000000000000000000..6bdff6fffa9b038190acdbc6f7b116097984799e --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 50, 0.5, 0.33669589463162186, 0.6620035767231229, 0.3672531694439306, 0.6438583284328754, 0.387978680454247, 0.6119417696967355, 0.4043694697511723, 0.5970221161648301]]} \ No newline at end of file diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/0_eval_0_79f998250d76be901393.mp4 b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/0_eval_0_79f998250d76be901393.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..d9eaefcbd93b6a5a58c1cf06406e4b66189f24f4 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/0_eval_0_79f998250d76be901393.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79f998250d76be90139304bc0b220414701797d2b24d46ca25335d7103e0f0eb +size 1134868 diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..263e43255d721a4044e74bb19f6b0b7fdf1474b8 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5777bf060ba6d7d77b457c7a7c8baf649e3ceefda782c804d6dae6ce969d0b1b +size 1129184 diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..611fc00bab9f6423719e09337a2aa83cc293dd9b --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a16b4848a19dfb39015158b10baf969d1b0db6dde103aec772d93c954a320f32 +size 1138006 diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..73bb8ff6b4f4b107be975911ce0be89b030a17f4 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:796eb7d92e1e7dcc5b92d8e7694013ab0e0cf7772c2da0a730543632ca0a7bd6 +size 1133100 diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..cebb051b7f8e8d6b47a6793e18e426b75b4d5279 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:227b80e6fd7f10383222c5cf88ae51a081022c5826eab44672ea2b7b2d6fadbb +size 1142083 diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..51d2fe73a74a8cb65b916717dee9e603268c76e5 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log @@ -0,0 +1,268 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736290847.008673]: resetting environment +[INFO] [1736290847.013721]: cleared current plan +[INFO] [1736290847.015192]: moving to home +[INFO] [1736290848.018763]: reached home +[INFO] [1736290848.020105]: reset action history +[INFO] [1736290850.726725]: environment reset +Starting episode 0 +[INFO] [1736290850.728860]: resetting environment +[INFO] [1736290850.731874]: cleared current plan +[INFO] [1736290850.732834]: moving to home +[INFO] [1736290851.735627]: reached home +[INFO] [1736290851.737435]: reset action history +[INFO] [1736290854.446450]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736290885.136009]: resetting environment +[INFO] [1736290885.139115]: cleared current plan +[INFO] [1736290885.139914]: moving to home +[INFO] [1736290886.141562]: reached home +[INFO] [1736290886.142996]: reset action history +[INFO] [1736290888.849733]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/episode_rosbags/episode_0_2025-01-07-18-01-28.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2206328.log +started roslaunch server http://158.130.50.37:45389/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2206595] +process[print_text-2]: started with pid [2206619] +process[rosbag_record-3]: started with pid [2206620] +started bagging! +For topic gripper_width: timestamp difference is 100227032 for nearest: 1736290904845485602 - target: 1736290904745258570 at idx 221 +For topic gripper_width: timestamp difference is 102165663 for nearest: 1736290907245483362 - target: 1736290907143317699 at idx 249 +For topic gripper_width: timestamp difference is 102333657 for nearest: 1736290907445485942 - target: 1736290907343152285 at idx 249 +For topic gripper_width: timestamp difference is 102999178 for nearest: 1736290908245482889 - target: 1736290908142483711 at idx 249 +[INFO] [1736290915.836694]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736290916.567966]: Stored episode 1. +[INFO] [1736290916.569513]: Saving video... +[INFO] [1736290916.751641]: Video saved! +Starting episode 1 +[INFO] [1736290916.768796]: resetting environment +[INFO] [1736290916.772603]: cleared current plan +[INFO] [1736290916.772853]: moving to home +[INFO] [1736290920.483957]: reached home +[INFO] [1736290920.485377]: reset action history +[INFO] [1736290923.193110]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290926.403202]: resetting environment +[INFO] [1736290926.405554]: cleared current plan +[INFO] [1736290926.405709]: moving to home +[INFO] [1736290927.407130]: reached home +[INFO] [1736290927.407533]: reset action history +[INFO] [1736290930.118480]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/episode_rosbags/episode_1_2025-01-07-18-02-10.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2206328.log +started roslaunch server http://158.130.50.37:40323/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2206735] +process[print_text-5]: started with pid [2206736] +process[rosbag_record-6]: started with pid [2206737] +started bagging! +[INFO] [1736290957.075835]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736290957.853264]: Stored episode 2. +[INFO] [1736290957.854219]: Saving video... +[INFO] [1736290958.440891]: Video saved! +Starting episode 2 +[INFO] [1736290958.460692]: resetting environment +[INFO] [1736290958.484373]: cleared current plan +[INFO] [1736290958.485009]: moving to home +[WARN] [1736290958.430448]: Plan exhausted +[WARN] [1736290958.471800]: Plan exhausted +[INFO] [1736290962.191109]: reached home +[INFO] [1736290962.192520]: reset action history +[INFO] [1736290964.987263]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736290982.859787]: resetting environment +[INFO] [1736290982.870312]: cleared current plan +[INFO] [1736290982.870743]: moving to home +[INFO] [1736290983.872576]: reached home +[INFO] [1736290983.873812]: reset action history +[INFO] [1736290986.582103]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/episode_rosbags/episode_2_2025-01-07-18-03-06.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2206328.log +started roslaunch server http://158.130.50.37:41067/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2206896] +process[print_text-8]: started with pid [2206897] +process[rosbag_record-9]: started with pid [2206898] +started bagging! +[INFO] [1736291013.696016]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736291014.511608]: Stored episode 3. +[INFO] [1736291014.513049]: Saving video... +[INFO] [1736291014.740976]: Video saved! +Starting episode 3 +[INFO] [1736291014.781612]: resetting environment +[INFO] [1736291014.798272]: cleared current plan +[INFO] [1736291014.806439]: moving to home +[INFO] [1736291018.513429]: reached home +[INFO] [1736291018.519902]: reset action history +[INFO] [1736291021.304899]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736291023.300875]: resetting environment +[INFO] [1736291023.309174]: cleared current plan +[INFO] [1736291023.309367]: moving to home +[INFO] [1736291024.311217]: reached home +[INFO] [1736291024.317652]: reset action history +[INFO] [1736291027.037304]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/episode_rosbags/episode_3_2025-01-07-18-03-47.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2206328.log +started roslaunch server http://158.130.50.37:45919/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2207008] +process[print_text-11]: started with pid [2207009] +process[rosbag_record-12]: started with pid [2207010] +started bagging! +For topic gripper_width: timestamp difference is -255643143 for nearest: 1736291041878762947 - target: 1736291042134406090 at idx 208 +[INFO] [1736291053.993948]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit[pub_text-10] killing on exit +[INFO] [1736291054.789335]: Stored episode 4. +[INFO] [1736291054.800211]: Saving video... +[INFO] [1736291055.050600]: Video saved! +Starting episode 4 +[INFO] [1736291055.094875]: resetting environment +[INFO] [1736291055.115016]: cleared current plan +[INFO] [1736291055.116215]: moving to home +[INFO] [1736291058.827737]: reached home +[INFO] [1736291058.834117]: reset action history +[INFO] [1736291061.614722]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736291072.770715]: resetting environment +[INFO] [1736291072.782665]: cleared current plan +[INFO] [1736291072.791493]: moving to home +[INFO] [1736291073.797396]: reached home +[INFO] [1736291073.800312]: reset action history +[INFO] [1736291076.579438]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/episode_rosbags/episode_4_2025-01-07-18-04-36.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2206328.log +started roslaunch server http://158.130.50.37:39783/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2207175] +process[print_text-14]: started with pid [2207176] +process[rosbag_record-15]: started with pid [2207177] +started bagging! +For topic gripper_width: timestamp difference is 115117288 for nearest: 1736291088545482374 - target: 1736291088430365086 at idx 166 +For topic gripper_width: timestamp difference is 116901381 for nearest: 1736291090745480521 - target: 1736291090628579140 at idx 195 +[INFO] [1736291103.718291]: Storing episode... +[rosbag_record-15] killing on exit +[print_text-14] killing on exit +[pub_text-13] killing on exit +[INFO] [1736291104.498656]: Stored episode 5. +[INFO] [1736291104.501091]: Saving video... +[INFO] [1736291104.908128]: Video saved! +[INFO] [1736291104.974142]: resetting environment +[INFO] [1736291104.992325]: cleared current plan +[INFO] [1736291104.995380]: moving to home +[INFO] [1736291108.704643]: reached home +[INFO] [1736291108.707446]: reset action history +[INFO] [1736291111.553842]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/requirements.txt b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-metadata.json b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..c94f1327623681688baaef5a1fe0bab143a1dd73 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T23:00:41.131479", + "startedAt": "2025-01-07T23:00:40.774898", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 4122.291499999999, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.584, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.364, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 796.296257019043 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..e61e74418aed19cb1a871e0545b8d685edf09c9b --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "79f998250d76be90139304bc0b220414701797d2b24d46ca25335d7103e0f0eb", "size": 1134868, "path": "media/videos/eval/0_eval_0_79f998250d76be901393.mp4"}, "global_step": 665, "_timestamp": 1736291120.5741582, "_runtime": 279.78978729248047, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "5777bf060ba6d7d77b457c7a7c8baf649e3ceefda782c804d6dae6ce969d0b1b", "size": 1129184, "path": "media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4"}, "eval/num_success": 5.0, "episode": 50.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.4496806859970093, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.4799937605857849, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6638181209564209, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 1.0, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "a16b4848a19dfb39015158b10baf969d1b0db6dde103aec772d93c954a320f32", "size": 1138006, "path": "media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "796eb7d92e1e7dcc5b92d8e7694013ab0e0cf7772c2da0a730543632ca0a7bd6", "size": 1133100, "path": "media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "227b80e6fd7f10383222c5cf88ae51a081022c5826eab44672ea2b7b2d6fadbb", "size": 1142083, "path": "media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "1b420c32327aa0ba7f7504ab8676a845cd69d26134af4445f97766065563f5e1", "size": 313, "artifact_path": "wandb-client-artifact://fkt6txu75o197xrztcvxz9imwac6tp1we2g6n3fba18ti2oh3dui1a7iv5gp0bjd39ih5qokv43utl7vxvta63gw2eojwiydbcx5upd34fu0o095aons7vd5ezetoyu9/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://4s9zrsjyg28srk5de0sj6zt4tbb9tkbgud2z5q5w44sr398yt4lqh0mcsq7vq3fth875qawrrlttt3e1dgz5oa28w4aqk2btn22ex55ec4521j0oh97ipmw1w46m6vpw:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 50, "total_success_rate": 0.5, "total_success_rate_lb_0.01": 0.33669589463162186, "total_success_rate_ub_0.01": 0.6620035767231229, "total_success_rate_lb_0.025": 0.3672531694439306, "total_success_rate_ub_0.025": 0.6438583284328754, "total_success_rate_lb_0.05": 0.387978680454247, "total_success_rate_ub_0.05": 0.6119417696967355, "total_success_rate_lb_0.1": 0.4043694697511723, "total_success_rate_ub_0.1": 0.5970221161648301, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "2f3f136f2937a5b3202d8da849c2ef52e3ba1205b3daba868e9420950bc47aef", "size": 505, "artifact_path": "wandb-client-artifact://h27qxyhk0in4n3lgn143gz4au3164i1er4ao2oia8rx7eyk6zhna105qp9uxvmm6nu9wah0h11g1kesu2zpeu2kwtbe0zd4uo8xhz92tdmtob1k36l4xe7xqlmk1c9gs/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://merbxh1pxa4r9jne8xjtqoztwnv5rqslms1w19bmpehrcedd8vl9kr59kk9ujjjhnfaeo68r8uthvjlih6xy4dognr22qnolz8lt5dpq05qd68c9qhqbhpbnezzysls8:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 297}, "eval/total_success_rate": 0.5, "eval/total_success_rate_lb_0.01": 0.33669590950012207, "eval/total_success_rate_ub_0.01": 0.6620035767555237, "eval/total_success_rate_lb_0.025": 0.3672531843185425, "eval/total_success_rate_ub_0.025": 0.6438583135604858, "eval/total_success_rate_lb_0.05": 0.3879786729812622, "eval/total_success_rate_ub_0.05": 0.6119417548179626, "eval/total_success_rate_lb_0.1": 0.4043694734573364, "eval/total_success_rate_ub_0.1": 0.5970221161842346} \ No newline at end of file diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug-internal.log b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5dfd1a1d795757374af03ce2b011908a2c084235 --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug-internal.log @@ -0,0 +1,810 @@ +2025-01-07 18:00:40,784 INFO StreamThr :2206495 [internal.py:wandb_internal():85] W&B internal server running at pid: 2206495, started at: 2025-01-07 18:00:40.784395 +2025-01-07 18:00:40,787 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:00:40,799 INFO WriterThread:2206495 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/run-kq00a7xd.wandb +2025-01-07 18:00:40,799 DEBUG SenderThread:2206495 [sender.py:send():379] send: header +2025-01-07 18:00:40,806 DEBUG SenderThread:2206495 [sender.py:send():379] send: run +2025-01-07 18:00:41,054 INFO SenderThread:2206495 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files +2025-01-07 18:00:41,054 INFO SenderThread:2206495 [sender.py:_start_run_threads():1188] run started: kq00a7xd with start time 1736290840.784371 +2025-01-07 18:00:41,068 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:00:41,069 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:00:41,113 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:00:41,118 DEBUG HandlerThread:2206495 [system_info.py:__init__():26] System info init +2025-01-07 18:00:41,118 DEBUG HandlerThread:2206495 [system_info.py:__init__():41] System info init done +2025-01-07 18:00:41,118 INFO HandlerThread:2206495 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:00:41,118 INFO SystemMonitor:2206495 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:00:41,118 INFO HandlerThread:2206495 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:00:41,119 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:00:41,120 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:00:41,120 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:00:41,121 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:00:41,121 INFO SystemMonitor:2206495 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:00:41,131 DEBUG HandlerThread:2206495 [system_info.py:probe():152] Probing system +2025-01-07 18:00:41,132 DEBUG HandlerThread:2206495 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:00:41,137 DEBUG HandlerThread:2206495 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:00:41,137 DEBUG HandlerThread:2206495 [system_info.py:probe():200] Probing system done +2025-01-07 18:00:41,137 DEBUG HandlerThread:2206495 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:00:41.131479', 'startedAt': '2025-01-07T23:00:40.774898', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 4122.291499999999, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.584, 'min': 800.0, 'max': 4700.0}, {'current': 4497.364, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 796.296257019043}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:00:41,137 INFO HandlerThread:2206495 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:00:41,137 INFO HandlerThread:2206495 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:00:41,137 DEBUG HandlerThread:2206495 [system_info.py:_save_code():44] Saving code +2025-01-07 18:00:41,139 DEBUG HandlerThread:2206495 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:00:41,139 DEBUG HandlerThread:2206495 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:00:41,200 DEBUG HandlerThread:2206495 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:00:41,201 INFO HandlerThread:2206495 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:00:41,204 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:00:41,204 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:00:41,204 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:00:41,204 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 18:00:41,433 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:00:41,433 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:00:41,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:41,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:00:41,435 DEBUG SenderThread:2206495 [sender.py:send():379] send: telemetry +2025-01-07 18:00:41,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:00:41,443 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/0c0rsn34-wandb-metadata.json +2025-01-07 18:00:41,469 INFO wandb-upload_2:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/uxde6usb-diff.patch +2025-01-07 18:00:41,477 INFO HandlerThread:2206495 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/tb" + save: true +} +control { + relay_id: "b4011873ceab42ec84f50f4786cd5445" +} +_info { + stream_id: "kq00a7xd" +} + +2025-01-07 18:00:41,477 DEBUG HandlerThread:2206495 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:00:41,524 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/23yb9kz3-code/FISH/eval_robot.py +2025-01-07 18:00:41,552 DEBUG SenderThread:2206495 [sender.py:send():379] send: tbrecord +2025-01-07 18:00:41,553 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:00:41,553 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 with policy live +2025-01-07 18:00:41,754 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/uyy54zab-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-metadata.json +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/diff.patch +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code/FISH/eval_robot.py +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/requirements.txt +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:42,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code/FISH +2025-01-07 18:00:42,057 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code +2025-01-07 18:00:42,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:43,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:44,055 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:44,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:45,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:46,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:46,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:47,008 DEBUG SenderThread:2206495 [sender.py:send():379] send: config +2025-01-07 18:00:47,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:48,056 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:48,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:49,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:50,057 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:50,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:51,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:51,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:52,057 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:52,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:53,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:54,058 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:54,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:55,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:56,059 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:00:56,433 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:00:56,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:00:56,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:56,552 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:00:57,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:58,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:00:59,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:00,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:01,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:02,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:02,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:03,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:04,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:05,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:06,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:07,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:07,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:08,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:09,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:10,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:11,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:01:11,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:01:11,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:12,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:12,573 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:13,063 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml +2025-01-07 18:01:13,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:14,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:15,439 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:16,439 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:17,439 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:18,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:18,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:19,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:20,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:21,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:22,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:23,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:24,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:24,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:25,440 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:26,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:01:26,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:01:26,441 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:27,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:28,067 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:28,441 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:29,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:30,068 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:30,122 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:30,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:31,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:32,069 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:32,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:33,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:34,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:35,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:35,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:36,442 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:37,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:38,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:39,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:40,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:41,121 DEBUG SystemMonitor:2206495 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:01:41,122 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:01:41,122 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:41,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:01:41,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:01:41,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:42,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:43,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:44,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:45,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:46,072 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:46,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:46,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:47,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:48,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:49,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:50,074 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:50,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:51,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:51,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:52,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:53,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:54,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:55,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:56,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:01:56,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:01:56,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:56,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:01:56,767 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:01:56,767 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_79f998250d76be901393.mp4 with policy now +2025-01-07 18:01:56,767 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:01:56,768 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:01:56,768 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:01:56,769 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:01:57,030 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/4bd47ty3-media/videos/eval/0_eval_0_79f998250d76be901393.mp4 +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/0_eval_0_79f998250d76be901393.mp4 +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media +2025-01-07 18:01:57,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:01:57,353 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/n69nlqr1-media/videos/eval/0_eval_0_79f998250d76be901393.mp4 +2025-01-07 18:01:57,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:58,076 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:01:58,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:01:59,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:00,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:01,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:02,077 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:02,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:02,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:03,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:04,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:05,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:06,079 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:06,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:07,079 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:02:07,347 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/tjryokvc-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:02:07,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:07,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:08,079 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:08,448 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:09,448 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:10,080 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:10,451 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:11,122 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:02:11,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:02:11,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:02:11,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:12,081 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:12,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:12,500 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:13,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:14,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:15,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:16,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:17,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:18,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:18,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:19,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:20,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:21,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:22,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:23,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:23,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:24,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:25,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:26,434 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:02:26,434 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:02:26,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:27,455 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:28,455 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:28,551 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:29,456 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:30,456 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:31,456 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:32,457 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:33,457 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:34,444 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:34,459 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:35,462 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:36,459 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:37,459 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:38,089 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:38,456 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:02:38,456 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 with policy now +2025-01-07 18:02:38,459 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:02:38,460 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:02:38,460 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:02:38,460 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:02:38,461 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:38,724 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/ofn66gmw-media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 +2025-01-07 18:02:39,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:02:39,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 +2025-01-07 18:02:39,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:02:39,378 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/6vngklpe-media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 +2025-01-07 18:02:39,460 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:39,510 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:40,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:40,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:41,090 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:41,123 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:02:41,443 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:02:41,443 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:02:41,489 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:42,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:43,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:44,091 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:44,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:44,988 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:45,465 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:46,092 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:02:46,465 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:47,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:48,465 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:49,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:50,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:50,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:51,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:52,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:53,466 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:54,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:55,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:02:55,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:56,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:02:56,436 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:02:56,481 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:57,467 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:58,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:02:59,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:00,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:00,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:01,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:02,468 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:03,098 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:03:03,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:03,924 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:03:03,924 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:03:03,924 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:03:04,098 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:03:04,098 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:04,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:05,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:05,925 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:06,099 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:06,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:07,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:08,100 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:08,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:09,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:10,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:10,994 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:11,123 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:03:11,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:03:11,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:03:11,477 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:12,479 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:13,482 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:14,482 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:15,482 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:16,445 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:16,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:17,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:18,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:19,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:20,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:21,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:21,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:22,483 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:23,485 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:24,485 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:25,485 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:26,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:03:26,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:03:26,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:26,561 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:27,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:28,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:29,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:30,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:31,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:32,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:32,486 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:33,487 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:34,490 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:34,771 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:03:34,771 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 with policy now +2025-01-07 18:03:34,774 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:03:34,774 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:03:34,774 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:03:34,775 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:03:35,044 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/1lad65yv-media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 +2025-01-07 18:03:35,109 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:03:35,109 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 +2025-01-07 18:03:35,109 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:03:35,403 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/jj1jq58q-media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 +2025-01-07 18:03:35,490 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:36,109 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:36,488 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:37,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:37,500 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:38,502 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:39,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:40,110 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:40,514 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:41,124 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:03:41,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:03:41,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:03:41,501 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:42,492 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:42,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:43,501 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:43,752 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:03:43,752 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:03:43,752 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:03:44,112 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:03:44,112 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:44,112 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:03:44,384 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/lqehp4vu-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:03:44,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:45,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:46,113 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:46,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:47,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:48,114 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:48,281 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:48,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:49,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:50,114 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:03:50,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:51,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:52,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:53,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:53,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:54,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:55,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:56,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:03:56,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:03:56,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:57,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:58,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:03:58,531 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:03:59,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:00,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:01,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:02,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:03,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:04,119 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:04,257 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:04,504 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:05,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:06,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:07,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:08,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:09,447 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:09,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:10,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:11,124 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:04:11,435 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:04:11,435 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:04:11,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:12,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:13,505 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:14,513 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:14,793 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:15,089 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:04:15,089 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 with policy now +2025-01-07 18:04:15,092 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:04:15,092 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:04:15,092 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:04:15,092 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:04:15,123 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:04:15,123 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 +2025-01-07 18:04:15,123 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:04:15,366 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/b3sruz0y-media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 +2025-01-07 18:04:15,510 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:15,641 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/fuqlh64b-media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 +2025-01-07 18:04:16,123 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:16,510 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:17,516 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:18,510 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:19,513 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:19,869 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:20,125 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:20,509 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:21,517 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:22,519 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:23,519 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:24,126 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:24,516 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:25,517 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:25,662 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:26,446 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:04:26,446 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:04:26,520 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:27,520 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:28,523 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:29,522 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:30,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:31,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:31,521 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:32,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:32,811 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:04:32,811 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:04:32,811 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:04:33,129 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:04:33,129 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:04:33,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:34,129 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:34,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:35,527 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:36,130 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:36,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:36,527 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:37,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:38,131 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:38,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:39,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:40,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:41,125 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:04:41,436 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:04:41,437 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:04:41,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:41,591 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:42,524 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:43,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:44,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:45,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:46,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:47,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:47,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:48,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:49,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:50,135 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:50,525 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:51,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:52,135 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:04:52,452 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:52,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:53,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:54,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:55,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:56,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:04:56,437 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:04:56,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:57,498 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:04:57,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:58,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:04:59,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:00,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:01,526 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:02,527 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:03,453 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:03,527 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:04,539 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:04,970 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:05:04,971 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 with policy now +2025-01-07 18:05:04,973 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:05:04,974 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:04,974 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:04,974 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:05,140 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:05,140 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 +2025-01-07 18:05:05,140 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval +2025-01-07 18:05:05,247 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/dfrn8fuh-media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 +2025-01-07 18:05:05,529 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/etj2slca-media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 +2025-01-07 18:05:05,538 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:06,140 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:06,534 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:07,536 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:08,469 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:08,543 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:09,536 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:10,142 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:10,537 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:11,125 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:05:11,438 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:05:11,439 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:05:11,540 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:12,541 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:13,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:13,601 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:14,143 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:14,556 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:15,552 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:16,559 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:17,548 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:18,548 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:19,133 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:19,145 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:19,428 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/9wvzz7mh-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:19,544 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:19,868 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:19,869 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:19,869 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,090 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:05:20,090 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:05:20,145 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:20,145 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:20,145 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:20,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:05:20,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media +2025-01-07 18:05:20,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval +2025-01-07 18:05:20,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table +2025-01-07 18:05:20,232 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:05:20,342 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/f853rohi-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:05:20,548 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:20,569 INFO wandb-upload_1:2206495 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmplfox_w0_ +2025-01-07 18:05:20,654 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:05:20,662 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:05:20,942 INFO SenderThread:2206495 [sender.py:send_request_log_artifact():1525] logged artifact run-kq00a7xd-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjA0ODY3Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTczNzE5OQ==', 'latestArtifact': None}} +2025-01-07 18:05:20,942 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:05:20,942 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json with policy now +2025-01-07 18:05:20,942 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:20,942 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,943 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,943 DEBUG SenderThread:2206495 [sender.py:send():379] send: telemetry +2025-01-07 18:05:20,943 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,947 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,947 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,947 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,948 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,948 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,948 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,948 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,949 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,949 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,949 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,949 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,950 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,950 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,950 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,950 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,951 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,951 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,951 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,951 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,952 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:20,952 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:20,952 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:05:21,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:21,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:21,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 18:05:21,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval +2025-01-07 18:05:21,211 INFO wandb-upload_2:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/1gg6gwtw-media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:05:21,323 INFO wandb-upload_1:2206495 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6ju784l7 +2025-01-07 18:05:21,357 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/k5py8w0s-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 18:05:21,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:21,725 INFO SenderThread:2206495 [sender.py:send_request_log_artifact():1525] logged artifact run-kq00a7xd-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjA0ODc3MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTczNzI3OA==', 'latestArtifact': None}} +2025-01-07 18:05:21,726 DEBUG SenderThread:2206495 [sender.py:send():379] send: files +2025-01-07 18:05:21,726 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json with policy now +2025-01-07 18:05:21,726 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:21,726 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:21,727 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:21,978 INFO wandb-upload_2:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/9gsncjd5-media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 18:05:22,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:22,146 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:22,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:23,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:24,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:24,728 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:25,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:26,437 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:05:26,437 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:05:26,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:27,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:28,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:29,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:30,503 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:30,545 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:31,150 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml +2025-01-07 18:05:31,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:32,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:33,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:34,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:35,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:36,457 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:36,546 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:37,547 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:38,581 DEBUG SenderThread:2206495 [sender.py:send():379] send: exit +2025-01-07 18:05:38,582 INFO SenderThread:2206495 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:05:38,582 INFO SenderThread:2206495 [sender.py:send_exit():588] handling runtime: 297 +2025-01-07 18:05:38,582 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:38,583 INFO SenderThread:2206495 [sender.py:send_exit():594] send defer +2025-01-07 18:05:38,583 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,583 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:05:38,583 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:38,583 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:05:38,583 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:05:38,583 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,583 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:05:38,583 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:38,583 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:05:38,584 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:05:38,584 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,584 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:05:38,584 INFO HandlerThread:2206495 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:05:38,584 DEBUG SystemMonitor:2206495 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:05:38,584 DEBUG SystemMonitor:2206495 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:05:38,584 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:05:38,585 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:05:38,654 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:05:38,655 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:05:38,655 INFO HandlerThread:2206495 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:05:38,655 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:38,655 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:05:38,655 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:05:38,655 DEBUG SenderThread:2206495 [sender.py:send():379] send: stats +2025-01-07 18:05:38,656 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,656 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:05:38,656 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:38,656 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:05:38,656 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:05:38,656 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:38,656 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:05:39,153 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:40,153 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:44,876 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:44,876 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:44,876 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:05:44,877 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:44,877 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:05:44,878 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:05:44,878 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:44,878 DEBUG SenderThread:2206495 [sender.py:send():379] send: history +2025-01-07 18:05:44,878 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:44,878 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:05:44,878 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:05:44,879 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:44,879 DEBUG SenderThread:2206495 [sender.py:send():379] send: summary +2025-01-07 18:05:44,879 INFO SenderThread:2206495 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:05:44,880 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:05:44,880 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:44,880 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:05:44,880 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:05:44,880 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:05:44,880 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:44,880 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:05:44,880 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:44,880 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:05:45,155 INFO Thread-12 :2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:45,583 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:46,623 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:05:46,623 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:46,624 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:46,624 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:05:46,624 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:46,624 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:05:46,624 INFO SenderThread:2206495 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:05:46,626 INFO SenderThread:2206495 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:05:46,628 INFO SenderThread:2206495 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:05:46,642 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:05:46,642 DEBUG SenderThread:2206495 [sender.py:send():379] send: artifact +2025-01-07 18:05:46,642 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:46,645 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:05:46,829 INFO SenderThread:2206495 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTg2NDkwOA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:05:46,829 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:46,829 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:05:46,829 INFO SenderThread:2206495 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml config.yaml +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json wandb-summary.json +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/requirements.txt requirements.txt +2025-01-07 18:05:47,157 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:05:47,158 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log output.log +2025-01-07 18:05:47,158 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/diff.patch diff.patch +2025-01-07 18:05:47,158 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 media/videos/eval/3_eval_3_796eb7d92e1e7dcc5b92.mp4 +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 media/videos/eval/2_eval_2_a16b4848a19dfb390151.mp4 +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/0_eval_0_79f998250d76be901393.mp4 media/videos/eval/0_eval_0_79f998250d76be901393.mp4 +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 media/videos/eval/1_eval_1_5777bf060ba6d7d77b45.mp4 +2025-01-07 18:05:47,160 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 media/videos/eval/4_eval_4_227b80e6fd7f10383222.mp4 +2025-01-07 18:05:47,166 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json media/table/eval/total_success_rate_ci_6_2f3f136f2937a5b3202d.table.json +2025-01-07 18:05:47,166 INFO SenderThread:2206495 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json media/table/eval/success_rate_ci_5_1b420c32327aa0ba7f75.table.json +2025-01-07 18:05:47,166 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:05:47,167 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,167 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:05:47,167 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,167 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:05:47,167 INFO SenderThread:2206495 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:05:47,339 INFO wandb-upload_0:2206495 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/config.yaml +2025-01-07 18:05:47,391 INFO wandb-upload_2:2206495 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/output.log +2025-01-07 18:05:47,391 INFO wandb-upload_1:2206495 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/wandb-summary.json +2025-01-07 18:05:47,392 INFO wandb-upload_4:2206495 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/files/requirements.txt +2025-01-07 18:05:47,434 INFO wandb-upload_3:2206495 [upload_job.py:push():130] Uploaded file /tmp/tmpl7qgw5g_wandb/375vrvdc-events.out.tfevents.1736290841.leonmkim-ROG-Strix-G15CS-G15CS.2206328.0 +2025-01-07 18:05:47,584 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:47,584 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:47,634 INFO Thread-11 (_thread_body):2206495 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:05:47,635 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,635 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:05:47,635 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,635 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:05:47,635 INFO SenderThread:2206495 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:05:47,636 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:05:47,637 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,637 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:05:47,637 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,637 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:05:47,637 INFO SenderThread:2206495 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:05:47,849 INFO SenderThread:2206495 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:05:47,849 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:05:47,849 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,849 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:05:47,849 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,849 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:05:47,849 INFO SenderThread:2206495 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:05:47,849 DEBUG SenderThread:2206495 [sender.py:send():379] send: final +2025-01-07 18:05:47,849 DEBUG SenderThread:2206495 [sender.py:send():379] send: footer +2025-01-07 18:05:47,849 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:05:47,850 INFO HandlerThread:2206495 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:05:47,850 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: defer +2025-01-07 18:05:47,850 INFO SenderThread:2206495 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:05:47,850 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:47,850 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:05:47,850 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:05:47,851 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:47,851 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:05:47,852 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:05:47,853 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:05:47,854 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:05:47,904 DEBUG SenderThread:2206495 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:05:47,904 INFO MainThread:2206495 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:05:47,905 INFO MainThread:2206495 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:05:47,906 INFO MainThread:2206495 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:05:47,906 DEBUG HandlerThread:2206495 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:05:47,906 INFO HandlerThread:2206495 [handler.py:finish():882] shutting down handler +2025-01-07 18:05:48,851 INFO WriterThread:2206495 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/run-kq00a7xd.wandb +2025-01-07 18:05:48,904 INFO SenderThread:2206495 [sender.py:finish():1615] shutting down sender +2025-01-07 18:05:48,904 INFO SenderThread:2206495 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:05:48,904 INFO SenderThread:2206495 [file_pusher.py:join():175] waiting for file pusher diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug.log b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..aaf2bf19add5c913aed1e16b2238cb40095bf93f --- /dev/null +++ b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Configure stats pid to 2206328 +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/settings +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:00:40,776 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:00:40,777 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug.log +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/logs/debug-internal.log +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 0, 'dataset_shuffle_seed': 1, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1022_0', 'wandb_notes': '1022_0_req_1348_0restarted_2', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 0, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': False, 'use_sdf_maps': False, 'use_normals_maps': False, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd'} +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:init():619] starting backend +2025-01-07 18:00:40,778 INFO MainThread:2206328 [wandb_init.py:init():623] setting up manager +2025-01-07 18:00:40,783 INFO MainThread:2206328 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:00:40,784 INFO MainThread:2206328 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:00:40,798 INFO MainThread:2206328 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:00:40,804 INFO MainThread:2206328 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:00:41,068 INFO MainThread:2206328 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:00:41,107 INFO MainThread:2206328 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:00:41,107 INFO MainThread:2206328 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:00:41,434 INFO MainThread:2206328 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:00:41,434 INFO MainThread:2206328 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:00:41,434 INFO MainThread:2206328 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:00:41,434 INFO MainThread:2206328 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:00:41,435 INFO MainThread:2206328 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:00:41,436 INFO MainThread:2206328 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1022_0/kq00a7xd/tb, True +2025-01-07 18:00:47,007 INFO MainThread:2206328 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:05:48,907 WARNING MsgRouterThr:2206328 [router.py:message_loop():77] message_loop has been closed diff --git a/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/run-kq00a7xd.wandb b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/run-kq00a7xd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..448035dcfd1a114d6a06ce2ebb3813e55bdfc23f Binary files /dev/null and b/kq00a7xd/wandb/run-20250107_180040-kq00a7xd/run-kq00a7xd.wandb differ diff --git a/snapshot_11499.pt b/snapshot_11499.pt new file mode 100644 index 0000000000000000000000000000000000000000..da75b2e2bca4f3a38b50fa57533db161a07ba158 --- /dev/null +++ b/snapshot_11499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87ff5bbd0f9b1400cb7665885600cb5bb541d86fcbadb5a6a174cf337afe0d3a +size 910102870 diff --git a/snapshot_7999.pt b/snapshot_7999.pt new file mode 100644 index 0000000000000000000000000000000000000000..8406519d5a26868e1420a81646c0b4b358eba583 --- /dev/null +++ b/snapshot_7999.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a3b9fcdd5cd1af2a1500d324027de02893a2835e2120a44adc57f97297ec7ed3 +size 910102360 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..518dd289cc86a147851c1e52db3c7cb4e696645e --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:04e35223c5cf69d90377800f069d2b10dadeb68fbbb2d2474a1d7b836e6c811a +size 910102360 diff --git a/vd7nl959/episode_rosbags/episode_0_2025-01-06-20-36-28.bag b/vd7nl959/episode_rosbags/episode_0_2025-01-06-20-36-28.bag new file mode 100644 index 0000000000000000000000000000000000000000..a57b157c7c4a095c723e2b73514ffa8a3ffcc6e8 --- /dev/null +++ b/vd7nl959/episode_rosbags/episode_0_2025-01-06-20-36-28.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c080336f6ef345494895e0717a207948f5cae1fd2f6d48e73082af970d5b1896 +size 1408695276 diff --git a/vd7nl959/episode_rosbags/episode_1_2025-01-06-20-37-08.bag b/vd7nl959/episode_rosbags/episode_1_2025-01-06-20-37-08.bag new file mode 100644 index 0000000000000000000000000000000000000000..efbee5c6c55e14545488f38393ad61e7c369fffe --- /dev/null +++ b/vd7nl959/episode_rosbags/episode_1_2025-01-06-20-37-08.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53f1a37ec395cf237bebee1733d9af5c8c1279343b23df1eb897333ab19f4f7d +size 1411749446 diff --git a/vd7nl959/episode_rosbags/episode_2_2025-01-06-20-38-01.bag b/vd7nl959/episode_rosbags/episode_2_2025-01-06-20-38-01.bag new file mode 100644 index 0000000000000000000000000000000000000000..f260e5bc97e52eecc58ecd44e5a48a244154c6c5 --- /dev/null +++ b/vd7nl959/episode_rosbags/episode_2_2025-01-06-20-38-01.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e359b3e10f6ae37715a4e6739c54f43df0355487c985c66a8ceb6b692598ce6f +size 1413716523 diff --git a/vd7nl959/episode_rosbags/episode_3_2025-01-06-20-38-45.bag b/vd7nl959/episode_rosbags/episode_3_2025-01-06-20-38-45.bag new file mode 100644 index 0000000000000000000000000000000000000000..f2b117e41224d0f994ed576af97f3ad655f8d400 --- /dev/null +++ b/vd7nl959/episode_rosbags/episode_3_2025-01-06-20-38-45.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4c11352691c12745de13939a3e63397042c1bbed9fd6a2fab48906d24f5c946 +size 1402242490 diff --git a/vd7nl959/episode_rosbags/episode_4_2025-01-06-20-39-25.bag b/vd7nl959/episode_rosbags/episode_4_2025-01-06-20-39-25.bag new file mode 100644 index 0000000000000000000000000000000000000000..0a8562758a671c127289a1e177b254a05915eebd --- /dev/null +++ b/vd7nl959/episode_rosbags/episode_4_2025-01-06-20-39-25.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74f34d2da05d2727c3cb3c4440026c6c3e6dafa10d278d7a8978ed39c4736c1 +size 1411180522