diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..690e9e903dec6b2bcbe5c8ab4d3fcf788e2280e1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,40 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 filter=lfs diff=lfs merge=lfs -text +q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 filter=lfs diff=lfs merge=lfs -text +q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 filter=lfs diff=lfs merge=lfs -text +q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 filter=lfs diff=lfs merge=lfs -text +q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 filter=lfs diff=lfs merge=lfs -text +q50r8fwa/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +q50r8fwa/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +q50r8fwa/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +q50r8fwa/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +q50r8fwa/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/episode_rosbags/episode_0_2025-01-07-20-21-32.bag filter=lfs diff=lfs merge=lfs -text +no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 filter=lfs diff=lfs merge=lfs -text +no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 filter=lfs diff=lfs merge=lfs -text +no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 filter=lfs diff=lfs merge=lfs -text +no798ka2/eval_video/3_eval.mp4 filter=lfs diff=lfs merge=lfs -text +no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 filter=lfs diff=lfs merge=lfs -text +no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 filter=lfs diff=lfs merge=lfs -text +no798ka2/eval_video/1_eval.mp4 filter=lfs diff=lfs merge=lfs -text +no798ka2/eval_video/2_eval.mp4 filter=lfs diff=lfs merge=lfs -text +no798ka2/eval_video/4_eval.mp4 filter=lfs diff=lfs merge=lfs -text +no798ka2/eval_video/0_eval.mp4 filter=lfs diff=lfs merge=lfs -text +w4b02c7s/episode_rosbags/episode_1_2025-01-07-20-22-04.bag filter=lfs diff=lfs merge=lfs -text +73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 filter=lfs diff=lfs merge=lfs -text +73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 filter=lfs diff=lfs merge=lfs -text +73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 filter=lfs diff=lfs merge=lfs -text +73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 filter=lfs diff=lfs merge=lfs -text +73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 filter=lfs diff=lfs merge=lfs -text diff --git a/73h7fwcu/episode_rosbags/aligned_depth_to_color_K.npy b/73h7fwcu/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/73h7fwcu/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/73h7fwcu/episode_rosbags/cam_tf_world.npy b/73h7fwcu/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/73h7fwcu/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/73h7fwcu/episode_rosbags/color_K.npy b/73h7fwcu/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/73h7fwcu/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/73h7fwcu/episode_rosbags/depth_K.npy b/73h7fwcu/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/73h7fwcu/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/73h7fwcu/eval_robot.log b/73h7fwcu/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..a6d444133fd12f8bf576fbbba2c459f101b41091 --- /dev/null +++ b/73h7fwcu/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 16:39:43,773][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 16:39:43,776][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/73h7fwcu/wandb/debug-internal.log b/73h7fwcu/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..29dd169160c2eeba1f59fac8aeb9d29d9373c19f --- /dev/null +++ b/73h7fwcu/wandb/debug-internal.log @@ -0,0 +1,851 @@ +2025-01-07 16:39:50,213 INFO StreamThr :2187469 [internal.py:wandb_internal():85] W&B internal server running at pid: 2187469, started at: 2025-01-07 16:39:50.213449 +2025-01-07 16:39:50,214 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status +2025-01-07 16:39:50,226 INFO WriterThread:2187469 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/run-73h7fwcu.wandb +2025-01-07 16:39:50,226 DEBUG SenderThread:2187469 [sender.py:send():379] send: header +2025-01-07 16:39:50,238 DEBUG SenderThread:2187469 [sender.py:send():379] send: run +2025-01-07 16:39:50,446 INFO SenderThread:2187469 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files +2025-01-07 16:39:50,446 INFO SenderThread:2187469 [sender.py:_start_run_threads():1188] run started: 73h7fwcu with start time 1736285990.213467 +2025-01-07 16:39:50,460 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 16:39:50,460 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: check_version +2025-01-07 16:39:50,535 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 16:39:50,540 DEBUG HandlerThread:2187469 [system_info.py:__init__():26] System info init +2025-01-07 16:39:50,540 DEBUG HandlerThread:2187469 [system_info.py:__init__():41] System info init done +2025-01-07 16:39:50,540 INFO HandlerThread:2187469 [system_monitor.py:start():194] Starting system monitor +2025-01-07 16:39:50,540 INFO SystemMonitor:2187469 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 16:39:50,540 INFO HandlerThread:2187469 [system_monitor.py:probe():214] Collecting system info +2025-01-07 16:39:50,541 INFO SystemMonitor:2187469 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 16:39:50,541 INFO SystemMonitor:2187469 [interfaces.py:start():188] Started disk monitoring +2025-01-07 16:39:50,542 INFO SystemMonitor:2187469 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 16:39:50,542 INFO SystemMonitor:2187469 [interfaces.py:start():188] Started memory monitoring +2025-01-07 16:39:50,543 INFO SystemMonitor:2187469 [interfaces.py:start():188] Started network monitoring +2025-01-07 16:39:50,553 DEBUG HandlerThread:2187469 [system_info.py:probe():152] Probing system +2025-01-07 16:39:50,554 DEBUG HandlerThread:2187469 [system_info.py:_probe_git():137] Probing git +2025-01-07 16:39:50,559 DEBUG HandlerThread:2187469 [system_info.py:_probe_git():145] Probing git done +2025-01-07 16:39:50,559 DEBUG HandlerThread:2187469 [system_info.py:probe():200] Probing system done +2025-01-07 16:39:50,559 DEBUG HandlerThread:2187469 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T21:39:50.553160', 'startedAt': '2025-01-07T21:39:50.204123', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3560.988625, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4496.073, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4495.306, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 687.2928199768066}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 16:39:50,559 INFO HandlerThread:2187469 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 16:39:50,559 INFO HandlerThread:2187469 [system_monitor.py:probe():227] Publishing system info +2025-01-07 16:39:50,559 DEBUG HandlerThread:2187469 [system_info.py:_save_code():44] Saving code +2025-01-07 16:39:50,565 DEBUG HandlerThread:2187469 [system_info.py:_save_code():67] Saving code done +2025-01-07 16:39:50,565 DEBUG HandlerThread:2187469 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 16:39:50,651 DEBUG HandlerThread:2187469 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 16:39:50,652 INFO HandlerThread:2187469 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 16:39:50,655 DEBUG SenderThread:2187469 [sender.py:send():379] send: files +2025-01-07 16:39:50,655 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 16:39:50,655 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 16:39:50,655 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 16:39:50,905 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 16:39:50,905 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: python_packages +2025-01-07 16:39:50,906 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:39:50,907 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:39:50,908 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/0w0wuxyz-wandb-metadata.json +2025-01-07 16:39:50,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:39:50,909 INFO HandlerThread:2187469 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/tb" + save: true +} +control { + relay_id: "eb2b166cfd3e47f1bd2aa1b619c9b04b" +} +_info { + stream_id: "73h7fwcu" +} + +2025-01-07 16:39:50,909 DEBUG HandlerThread:2187469 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 16:39:50,910 INFO wandb-upload_2:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/fgeyn7jv-diff.patch +2025-01-07 16:39:50,913 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/2r4o9c9h-code/FISH/eval_robot.py +2025-01-07 16:39:50,995 DEBUG SenderThread:2187469 [sender.py:send():379] send: telemetry +2025-01-07 16:39:50,996 DEBUG SenderThread:2187469 [sender.py:send():379] send: tbrecord +2025-01-07 16:39:50,996 DEBUG SenderThread:2187469 [sender.py:send():379] send: files +2025-01-07 16:39:50,996 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 with policy live +2025-01-07 16:39:51,170 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/igt2f9tp-events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:39:51,448 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:39:51,448 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/requirements.txt +2025-01-07 16:39:51,448 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/code/FISH/eval_robot.py +2025-01-07 16:39:51,448 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-metadata.json +2025-01-07 16:39:51,448 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/diff.patch +2025-01-07 16:39:51,448 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:39:51,449 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/code +2025-01-07 16:39:51,449 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/code/FISH +2025-01-07 16:39:51,906 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:39:52,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:39:53,447 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:39:53,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:39:54,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:39:55,456 DEBUG SenderThread:2187469 [sender.py:send():379] send: config +2025-01-07 16:39:55,456 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:39:55,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:39:56,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:39:57,448 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:39:57,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:39:58,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:39:59,449 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:39:59,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:00,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:00,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:01,449 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:01,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:02,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:03,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:04,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:05,451 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:05,906 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:40:05,906 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:40:05,949 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:06,038 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:06,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:07,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:08,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:09,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:10,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:11,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:11,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:12,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:13,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:14,910 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:15,453 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:15,910 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:16,911 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:16,950 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:17,911 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:18,911 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:19,454 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:19,911 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:20,906 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:40:20,906 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:40:20,953 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:21,911 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:21,974 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:22,455 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/config.yaml +2025-01-07 16:40:22,911 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:23,912 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:24,912 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:25,912 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:26,912 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:27,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:27,912 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:28,913 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:29,913 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:30,913 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:31,913 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:32,913 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:33,458 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:33,693 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:33,913 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:34,913 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:35,906 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:40:35,906 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:40:35,949 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:36,914 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:37,459 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:37,914 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:38,914 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:39,460 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:39,590 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:39,914 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:40,914 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:41,914 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:42,914 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:43,914 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:44,908 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:44,914 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:45,915 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:46,915 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:47,462 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:47,915 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:48,915 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:49,916 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:50,294 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:50,543 DEBUG SystemMonitor:2187469 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 16:40:50,544 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:40:50,906 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:40:50,907 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:40:50,953 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:51,916 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:52,772 DEBUG SenderThread:2187469 [sender.py:send():379] send: files +2025-01-07 16:40:52,772 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 with policy now +2025-01-07 16:40:52,772 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:40:52,773 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:40:52,773 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:40:52,773 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:40:52,916 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:52,991 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/g6ivmvik-media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 +2025-01-07 16:40:53,463 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:53,464 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:40:53,464 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 +2025-01-07 16:40:53,464 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media +2025-01-07 16:40:53,464 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos +2025-01-07 16:40:53,464 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval +2025-01-07 16:40:53,729 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/y9m0poix-media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 +2025-01-07 16:40:53,916 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:54,916 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:55,464 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:55,601 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:40:55,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:56,916 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:57,465 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:57,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:58,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:40:59,466 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:40:59,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:00,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:00,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:01,466 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:41:01,701 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/ianv1k7a-events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:41:01,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:02,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:03,467 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:03,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:04,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:05,906 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:41:05,907 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:41:05,918 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:05,965 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:06,918 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:07,469 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:07,918 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:08,918 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:09,918 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:10,918 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:11,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:11,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:12,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:13,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:14,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:15,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:16,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:16,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:17,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:18,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:19,920 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:20,550 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:41:20,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:41:20,907 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:41:20,949 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:21,920 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:22,047 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:22,920 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:23,920 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:24,921 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:25,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:26,922 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:27,909 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:27,922 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:28,922 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:29,922 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:30,922 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:31,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:32,910 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:32,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:33,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:34,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:35,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:41:35,907 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:41:35,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:36,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:37,478 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:37,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:38,047 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:38,924 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:39,924 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:40,928 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:41,124 DEBUG SenderThread:2187469 [sender.py:send():379] send: files +2025-01-07 16:41:41,124 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 with policy now +2025-01-07 16:41:41,127 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:41:41,128 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:41:41,128 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:41:41,140 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:41:41,346 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/jq9x5y2k-media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 +2025-01-07 16:41:41,479 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:41,479 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:41:41,480 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 +2025-01-07 16:41:41,480 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval +2025-01-07 16:41:41,736 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/8to2aeel-media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 +2025-01-07 16:41:41,927 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:42,927 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:43,153 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:43,480 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:43,926 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:44,928 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:45,481 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:45,935 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:46,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:47,929 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:48,397 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:48,929 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:49,482 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:49,929 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:50,545 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:41:50,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:41:50,907 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:41:50,929 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:51,351 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:41:51,351 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:41:51,351 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:41:51,483 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:41:51,483 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:41:51,929 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:52,929 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:53,484 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:53,773 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:53,930 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:54,929 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:55,485 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:55,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:56,930 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:57,486 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:41:57,930 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:58,910 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:41:58,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:41:59,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:00,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:01,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:02,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:03,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:04,289 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:04,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:05,488 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:42:05,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:42:05,907 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:42:05,949 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:06,931 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:07,932 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:08,932 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:09,932 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:09,969 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:10,932 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:11,932 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:12,932 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:13,932 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:14,932 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:15,492 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:42:15,521 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:15,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:16,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:17,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:18,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:19,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:20,545 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:42:20,546 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:20,907 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:42:20,907 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:42:20,949 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:21,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:22,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:23,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:24,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:25,911 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:25,934 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:26,934 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:27,934 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:28,935 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:29,942 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:30,853 DEBUG SenderThread:2187469 [sender.py:send():379] send: files +2025-01-07 16:42:30,853 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 with policy now +2025-01-07 16:42:30,856 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:42:30,857 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:42:30,857 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:42:30,863 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:42:30,911 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:30,935 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:31,078 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/h1bikttj-media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 +2025-01-07 16:42:31,498 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:42:31,498 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:42:31,498 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 +2025-01-07 16:42:31,498 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval +2025-01-07 16:42:31,776 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/iq9oyjsb-media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 +2025-01-07 16:42:31,938 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:32,945 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:33,498 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:42:33,941 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:34,943 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:35,499 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:42:35,913 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:35,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:42:35,933 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:42:35,981 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:36,942 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:37,943 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:38,945 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:39,500 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:42:39,951 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:40,921 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:40,949 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:41,948 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:42,949 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:43,946 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:44,951 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:45,921 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:45,957 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:46,961 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:47,955 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:48,959 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:49,960 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:50,546 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:42:50,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:42:50,917 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:42:50,961 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:51,048 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:51,950 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:52,957 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:53,959 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:54,951 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:55,956 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:56,930 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:42:56,962 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:57,965 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:58,966 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:42:59,970 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:00,970 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:01,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:01,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:02,968 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:03,977 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:04,968 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:05,915 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:43:05,916 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:43:05,971 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:06,971 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:07,066 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:07,970 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:08,972 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:09,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:10,196 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:43:10,197 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:43:10,197 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:43:10,511 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:43:10,511 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:43:10,758 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/wbn6iz9r-events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:43:10,978 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:11,511 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:43:11,974 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:12,225 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:12,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:13,512 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:43:13,974 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:14,972 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:15,512 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:43:15,972 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:16,972 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:17,513 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:43:17,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:17,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:18,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:19,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:20,546 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:43:20,915 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:43:20,915 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:43:20,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:21,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:22,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:23,041 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:23,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:24,973 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:25,974 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:26,974 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:27,974 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:28,924 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:28,974 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:29,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:30,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:31,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:32,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:33,924 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:33,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:34,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:35,915 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:43:35,915 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:43:35,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:36,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:37,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:38,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:39,014 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:39,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:40,976 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:41,522 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:43:41,976 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:42,976 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:43,523 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:43:43,976 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:44,193 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:44,976 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:45,976 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:46,976 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:47,976 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:48,977 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:49,224 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:49,984 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:50,548 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:43:50,614 DEBUG SenderThread:2187469 [sender.py:send():379] send: files +2025-01-07 16:43:50,615 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 with policy now +2025-01-07 16:43:50,618 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:43:50,619 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:43:50,619 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:43:50,619 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:43:50,825 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/md730ny6-media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 +2025-01-07 16:43:50,915 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:43:50,916 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:43:50,977 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:51,525 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:43:51,525 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:43:51,526 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 +2025-01-07 16:43:51,526 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval +2025-01-07 16:43:51,799 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/plzawn72-media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 +2025-01-07 16:43:51,989 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:52,986 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:53,987 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:54,225 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:43:55,010 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:55,527 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:43:55,998 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:57,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:57,997 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:59,001 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:43:59,528 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:43:59,998 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:00,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:01,004 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:02,001 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:02,998 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:03,999 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:05,012 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:05,923 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:44:05,923 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:44:06,008 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:06,059 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:07,009 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:08,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:09,013 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:10,015 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:11,016 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:11,938 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:12,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:12,532 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:44:12,758 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/io20jw90-events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:44:13,017 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:13,270 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:44:13,270 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:44:13,270 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:44:13,533 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:44:13,533 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:44:13,533 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:44:14,018 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:15,020 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:15,533 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:44:16,026 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:17,020 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:17,534 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:44:17,826 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:18,020 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:19,020 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:19,535 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:44:20,020 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:20,548 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:44:20,916 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:44:20,917 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:44:21,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:22,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:23,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:23,035 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:24,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:25,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:26,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:27,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:28,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:28,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:29,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:30,021 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:31,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:31,539 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:44:32,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:33,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:33,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:34,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:35,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:35,917 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:44:35,917 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:44:36,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:37,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:38,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:39,020 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:39,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:40,022 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:41,023 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:42,023 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:43,023 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:44,023 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:44,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:45,023 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:45,544 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:44:46,023 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:47,023 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:48,024 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:49,024 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:49,933 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:50,024 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:50,548 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:44:50,916 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:44:50,917 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:44:51,024 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:52,032 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:52,877 DEBUG SenderThread:2187469 [sender.py:send():379] send: files +2025-01-07 16:44:52,877 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 with policy now +2025-01-07 16:44:52,886 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:44:52,886 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:44:52,886 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:44:52,886 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:44:53,030 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:53,101 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/s3czuywt-media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 +2025-01-07 16:44:53,547 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:44:53,547 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:44:53,547 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 +2025-01-07 16:44:53,547 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval +2025-01-07 16:44:53,808 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/yr7dugz0-media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 +2025-01-07 16:44:54,033 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:55,031 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:55,548 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:44:55,915 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:44:56,031 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:57,032 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:57,548 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:44:58,038 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:59,049 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:44:59,549 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:45:00,047 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:00,951 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:45:01,051 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:02,049 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:03,051 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:04,048 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:05,056 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:05,924 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 16:45:05,924 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: stop_status +2025-01-07 16:45:06,002 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:45:06,048 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:07,070 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:08,047 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:08,073 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:45:08,073 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:45:08,215 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:45:08,515 INFO wandb-upload_0:2187469 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpobk1wxpy +2025-01-07 16:45:08,553 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:45:08,553 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-07 16:45:08,553 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media +2025-01-07 16:45:08,553 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval +2025-01-07 16:45:08,554 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table +2025-01-07 16:45:08,638 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 16:45:08,650 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 16:45:08,882 INFO SenderThread:2187469 [sender.py:send_request_log_artifact():1525] logged artifact run-73h7fwcu-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTgwODM0MA==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY2NjI3OA==', 'latestArtifact': None}} +2025-01-07 16:45:08,882 DEBUG SenderThread:2187469 [sender.py:send():379] send: files +2025-01-07 16:45:08,882 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json with policy now +2025-01-07 16:45:08,882 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:45:08,882 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,883 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,883 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:45:08,883 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,887 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,887 DEBUG SenderThread:2187469 [sender.py:send():379] send: telemetry +2025-01-07 16:45:08,887 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,887 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,887 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,888 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,888 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,888 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,888 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,889 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,889 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,889 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,889 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,890 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,890 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,890 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,890 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,891 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,891 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,891 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,891 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,892 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,892 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:08,892 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:08,892 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 16:45:09,047 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:09,052 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/y00wa00n-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-07 16:45:09,289 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/8vgp9ckq-media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-07 16:45:09,376 INFO wandb-upload_1:2187469 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmphpi6vprz +2025-01-07 16:45:09,553 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:45:09,553 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:45:09,553 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:45:09,553 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json +2025-01-07 16:45:09,554 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval +2025-01-07 16:45:09,754 INFO SenderThread:2187469 [sender.py:send_request_log_artifact():1525] logged artifact run-73h7fwcu-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNTgwODM2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTY2NjI4NQ==', 'latestArtifact': None}} +2025-01-07 16:45:09,754 DEBUG SenderThread:2187469 [sender.py:send():379] send: files +2025-01-07 16:45:09,754 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json with policy now +2025-01-07 16:45:09,754 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:45:09,754 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:09,754 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:09,920 INFO wandb-upload_2:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/x9wfgo8k-media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json +2025-01-07 16:45:10,047 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:10,361 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/g9aze1ze-media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json +2025-01-07 16:45:10,553 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:45:11,047 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:11,554 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:45:11,938 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:45:12,047 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:12,554 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/config.yaml +2025-01-07 16:45:12,917 DEBUG SenderThread:2187469 [sender.py:send():379] send: exit +2025-01-07 16:45:12,917 INFO SenderThread:2187469 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 16:45:12,917 INFO SenderThread:2187469 [sender.py:send_exit():588] handling runtime: 322 +2025-01-07 16:45:12,918 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:12,918 INFO SenderThread:2187469 [sender.py:send_exit():594] send defer +2025-01-07 16:45:12,918 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:12,918 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 16:45:12,918 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:12,918 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 16:45:12,918 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 1 +2025-01-07 16:45:12,918 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:12,918 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 16:45:12,919 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:12,919 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 16:45:12,919 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 2 +2025-01-07 16:45:12,919 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:12,919 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 16:45:12,919 INFO HandlerThread:2187469 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 16:45:12,919 DEBUG SystemMonitor:2187469 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 16:45:12,919 DEBUG SystemMonitor:2187469 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 16:45:12,920 INFO HandlerThread:2187469 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 16:45:12,920 INFO HandlerThread:2187469 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 16:45:12,973 INFO HandlerThread:2187469 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 16:45:12,974 INFO HandlerThread:2187469 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 16:45:12,974 INFO HandlerThread:2187469 [interfaces.py:finish():200] Joined network monitor +2025-01-07 16:45:12,974 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:12,974 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 16:45:12,974 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 3 +2025-01-07 16:45:12,974 DEBUG SenderThread:2187469 [sender.py:send():379] send: stats +2025-01-07 16:45:12,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:12,975 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 16:45:12,975 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:12,975 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 16:45:12,975 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 4 +2025-01-07 16:45:12,975 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:12,975 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 16:45:13,555 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:45:15,556 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:45:19,339 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:45:19,339 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:45:19,339 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 16:45:19,340 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:19,340 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 16:45:19,340 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 5 +2025-01-07 16:45:19,340 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:45:19,341 DEBUG SenderThread:2187469 [sender.py:send():379] send: history +2025-01-07 16:45:19,341 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: summary_record +2025-01-07 16:45:19,341 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:19,341 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 16:45:19,341 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:19,342 DEBUG SenderThread:2187469 [sender.py:send():379] send: summary +2025-01-07 16:45:19,345 INFO SenderThread:2187469 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 16:45:19,345 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:19,345 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 16:45:19,346 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 6 +2025-01-07 16:45:19,346 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:19,346 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 16:45:19,346 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:19,346 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 16:45:19,346 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 7 +2025-01-07 16:45:19,346 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 16:45:19,346 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:19,346 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 16:45:19,346 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:19,346 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 16:45:19,557 INFO Thread-12 :2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:45:19,918 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:45:23,253 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 8 +2025-01-07 16:45:23,254 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:45:23,254 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:23,254 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 16:45:23,254 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:23,254 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 16:45:23,254 INFO SenderThread:2187469 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 16:45:23,257 INFO SenderThread:2187469 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 16:45:23,258 INFO SenderThread:2187469 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 16:45:23,275 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 9 +2025-01-07 16:45:23,276 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:23,276 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 16:45:23,276 DEBUG SenderThread:2187469 [sender.py:send():379] send: artifact +2025-01-07 16:45:23,497 INFO SenderThread:2187469 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'versionIndex': 432}}} +2025-01-07 16:45:23,497 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:23,497 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 16:45:23,497 INFO SenderThread:2187469 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 16:45:23,558 INFO SenderThread:2187469 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:45:23,559 INFO SenderThread:2187469 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files +2025-01-07 16:45:23,559 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/config.yaml config.yaml +2025-01-07 16:45:23,559 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json wandb-summary.json +2025-01-07 16:45:23,561 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/requirements.txt requirements.txt +2025-01-07 16:45:23,561 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-metadata.json wandb-metadata.json +2025-01-07 16:45:23,562 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:45:23,562 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log output.log +2025-01-07 16:45:23,562 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/diff.patch diff.patch +2025-01-07 16:45:23,566 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 16:45:23,570 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 +2025-01-07 16:45:23,570 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 +2025-01-07 16:45:23,571 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 +2025-01-07 16:45:23,571 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 +2025-01-07 16:45:23,571 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 +2025-01-07 16:45:23,571 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json +2025-01-07 16:45:23,571 INFO SenderThread:2187469 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json +2025-01-07 16:45:23,571 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 10 +2025-01-07 16:45:23,571 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:23,571 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 16:45:23,572 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:23,572 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 16:45:23,572 INFO SenderThread:2187469 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:45:23,724 INFO wandb-upload_2:2187469 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/config.yaml +2025-01-07 16:45:23,730 INFO wandb-upload_0:2187469 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/requirements.txt +2025-01-07 16:45:23,769 INFO wandb-upload_1:2187469 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json +2025-01-07 16:45:23,795 INFO wandb-upload_4:2187469 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log +2025-01-07 16:45:23,903 INFO wandb-upload_3:2187469 [upload_job.py:push():130] Uploaded file /tmp/tmpdr5tofvbwandb/31evxjh8-events.out.tfevents.1736285990.leonmkim-ROG-Strix-G15CS-G15CS.2187297.0 +2025-01-07 16:45:23,920 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:45:23,920 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:45:24,104 INFO Thread-11 (_thread_body):2187469 [sender.py:transition_state():614] send defer: 11 +2025-01-07 16:45:24,104 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:24,104 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 16:45:24,104 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:24,104 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 16:45:24,104 INFO SenderThread:2187469 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 16:45:24,106 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 12 +2025-01-07 16:45:24,106 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:24,106 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 16:45:24,107 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:24,107 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 16:45:24,107 INFO SenderThread:2187469 [file_stream.py:finish():601] file stream finish called +2025-01-07 16:45:24,326 INFO SenderThread:2187469 [file_stream.py:finish():605] file stream finish is done +2025-01-07 16:45:24,326 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 13 +2025-01-07 16:45:24,327 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:24,327 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 16:45:24,327 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:24,327 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 16:45:24,327 INFO SenderThread:2187469 [sender.py:transition_state():614] send defer: 14 +2025-01-07 16:45:24,327 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: defer +2025-01-07 16:45:24,327 INFO HandlerThread:2187469 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 16:45:24,328 DEBUG SenderThread:2187469 [sender.py:send():379] send: final +2025-01-07 16:45:24,328 DEBUG SenderThread:2187469 [sender.py:send():379] send: footer +2025-01-07 16:45:24,328 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: defer +2025-01-07 16:45:24,328 INFO SenderThread:2187469 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 16:45:24,328 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:45:24,328 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 16:45:24,329 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:45:24,329 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: server_info +2025-01-07 16:45:24,329 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 16:45:24,331 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 16:45:24,331 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 16:45:24,332 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 16:45:24,382 DEBUG SenderThread:2187469 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 16:45:24,383 INFO MainThread:2187469 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 16:45:24,383 INFO MainThread:2187469 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 16:45:24,383 INFO MainThread:2187469 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 16:45:24,384 DEBUG HandlerThread:2187469 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 16:45:24,384 INFO HandlerThread:2187469 [handler.py:finish():882] shutting down handler +2025-01-07 16:45:25,332 INFO WriterThread:2187469 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/run-73h7fwcu.wandb +2025-01-07 16:45:25,383 INFO SenderThread:2187469 [sender.py:finish():1615] shutting down sender +2025-01-07 16:45:25,383 INFO SenderThread:2187469 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 16:45:25,383 INFO SenderThread:2187469 [file_pusher.py:join():175] waiting for file pusher diff --git a/73h7fwcu/wandb/debug.log b/73h7fwcu/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..47a69999f9981d83fba4a7247f17f9edd64c6a7a --- /dev/null +++ b/73h7fwcu/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 16:39:50,207 INFO MainThread:2187297 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 16:39:50,207 INFO MainThread:2187297 [wandb_setup.py:_flush():76] Configure stats pid to 2187297 +2025-01-07 16:39:50,207 INFO MainThread:2187297 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 16:39:50,207 INFO MainThread:2187297 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/settings +2025-01-07 16:39:50,207 INFO MainThread:2187297 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 16:39:50,207 INFO MainThread:2187297 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 16:39:50,207 INFO MainThread:2187297 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 16:39:50,207 INFO MainThread:2187297 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 16:39:50,207 INFO MainThread:2187297 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/logs/debug.log +2025-01-07 16:39:50,208 INFO MainThread:2187297 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/logs/debug-internal.log +2025-01-07 16:39:50,208 INFO MainThread:2187297 [wandb_init.py:init():569] calling init triggers +2025-01-07 16:39:50,208 INFO MainThread:2187297 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu'} +2025-01-07 16:39:50,208 INFO MainThread:2187297 [wandb_init.py:init():619] starting backend +2025-01-07 16:39:50,208 INFO MainThread:2187297 [wandb_init.py:init():623] setting up manager +2025-01-07 16:39:50,212 INFO MainThread:2187297 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 16:39:50,213 INFO MainThread:2187297 [wandb_init.py:init():631] backend started and connected +2025-01-07 16:39:50,226 INFO MainThread:2187297 [wandb_init.py:init():720] updated telemetry +2025-01-07 16:39:50,234 INFO MainThread:2187297 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 16:39:50,460 INFO MainThread:2187297 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 16:39:50,531 INFO MainThread:2187297 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 16:39:50,531 INFO MainThread:2187297 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 16:39:50,906 INFO MainThread:2187297 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 16:39:50,906 INFO MainThread:2187297 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 16:39:50,906 INFO MainThread:2187297 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 16:39:50,907 INFO MainThread:2187297 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 16:39:50,908 INFO MainThread:2187297 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 16:39:50,908 INFO MainThread:2187297 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/tb, True +2025-01-07 16:39:55,456 INFO MainThread:2187297 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'hbm', 'left_book_slot': 'twodim'} +2025-01-07 16:45:25,384 WARNING MsgRouterThr:2187297 [router.py:message_loop():77] message_loop has been closed diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/config.yaml b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3730a9c6c84eb6d2583607e524034e5ee6000df --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1013_1' +wandb_notes: + desc: null + value: 1013_1_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736285990 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: hbm +left_book_slot: + desc: null + value: twodim diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/diff.patch b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..8adda0290b36965f42f4119e05fd13ea9507158f --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/diff.patch @@ -0,0 +1,15 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..1c0be7a 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -139,8 +139,8 @@ load_checkpoint: ${agent.load_checkpoint} + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' +-# wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json new file mode 100644 index 0000000000000000000000000000000000000000..faa9910a25d3f0826c10a7b81de69b8b94278cd1 --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.0, 0, 0.5363743901179997, 0, 0.48415964842159465, 0, 0.21277225023292168, 0, 0.1673506349990479]]} \ No newline at end of file diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json new file mode 100644 index 0000000000000000000000000000000000000000..4d6759ce0ae6d9b968189777f76dc2e57b3ada74 --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[25, 40, 0.625, 0.4539579749199444, 0.7813661097917493, 0.4635710865332028, 0.7571975811801546, 0.4847957044870268, 0.7443524449575001, 0.5340144000879443, 0.7229229919166562]]} \ No newline at end of file diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..04c91440811ec3b6b6ab8b086e2e80600e53484a --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67a649dac0864a0b9776f767bc7576b313825aedd1fb1e5bd8232dcfb9a9c3be +size 1465999 diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..1aa562d50f43229e1a28b94449af61a28a56b16c --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cffb0391d794a1519dbaf82e22d4d4b11d19cd4cb117756b1dd14997b708132a +size 1495191 diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..93bb193f1aef69047c70b4402b3d856c529d5d76 --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43ad98f5bb65a23f42a77ee3e7485ca244a84f2eca11d653428a694f8dbc7587 +size 1523457 diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..fa2e4b85c985a6458f8ef2591294c19d81c85228 --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5b62306ad8696d8b8c905da27e65c2d240f6e4906b7c2a1fcbe93a1da19fac61 +size 1485499 diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5ff7771a79af90c3423be22b4323ef27e397cca6 --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66fc966ceba4d62103d365426065720163bd13fbbc1c8e542447c5b4dabe78dd +size 1502654 diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..ea4773bd1e136657484159b4085aa0bd61740d42 --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/output.log @@ -0,0 +1,290 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736285995.457175]: resetting environment +[INFO] [1736285995.465323]: cleared current plan +[INFO] [1736285995.467233]: moving to home +[INFO] [1736285996.471064]: reached home +[INFO] [1736285996.472350]: reset action history +[INFO] [1736285999.181207]: environment reset +Starting episode 0 +[INFO] [1736285999.185618]: resetting environment +[INFO] [1736285999.192549]: cleared current plan +[INFO] [1736285999.193753]: moving to home +[INFO] [1736286000.197190]: reached home +[INFO] [1736286000.198450]: reset action history +[INFO] [1736286002.908808]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736286013.072206]: resetting environment +[INFO] [1736286013.076876]: cleared current plan +[INFO] [1736286013.078669]: moving to home +[INFO] [1736286014.082706]: reached home +[INFO] [1736286014.084465]: reset action history +[INFO] [1736286016.793408]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/episode_rosbags/episode_0_2025-01-07-16-40-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2187297.log +started roslaunch server http://158.130.50.37:35775/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2187576] +process[print_text-2]: started with pid [2187600] +process[rosbag_record-3]: started with pid [2187601] +started bagging! +For topic gripper_width: timestamp difference is 129801058 for nearest: 1736286031645529293 - target: 1736286031515728235 at idx 203 +For topic gripper_width: timestamp difference is 100160986 for nearest: 1736286036078889996 - target: 1736286035978729010 at idx 249 +For topic gripper_width: timestamp difference is 100523277 for nearest: 1736286036578863903 - target: 1736286036478340626 at idx 249 +For topic gripper_width: timestamp difference is 107315137 for nearest: 1736286044945530488 - target: 1736286044838215351 at idx 249 +For topic gripper_width: timestamp difference is 108376987 for nearest: 1736286046245555408 - target: 1736286046137178421 at idx 249 +[INFO] [1736286051.770073]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit[pub_text-1] killing on exit +[INFO] [1736286052.541237]: Stored episode 1. +[INFO] [1736286052.541545]: Saving video... +[INFO] [1736286052.741593]: Video saved! +Starting episode 1 +[INFO] [1736286052.777390]: resetting environment +[INFO] [1736286052.793707]: cleared current plan +[INFO] [1736286052.797335]: moving to home +[INFO] [1736286055.600956]: reached home +[INFO] [1736286055.601413]: reset action history +[INFO] [1736286058.313222]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736286061.123260]: resetting environment +[INFO] [1736286061.127646]: cleared current plan +[INFO] [1736286061.127831]: moving to home +[INFO] [1736286062.129265]: reached home +[INFO] [1736286062.129676]: reset action history +[INFO] [1736286064.842000]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/episode_rosbags/episode_1_2025-01-07-16-41-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2187297.log +started roslaunch server http://158.130.50.37:44223/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2187722] +process[print_text-5]: started with pid [2187723] +process[rosbag_record-6]: started with pid [2187747] +started bagging! +For topic gripper_width: timestamp difference is 115052639 for nearest: 1736286095845565973 - target: 1736286095730513334 at idx 249 +[INFO] [1736286099.701087]: Storing episode... +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[print_text-5] killing on exit +[INFO] [1736286100.413486]: Stored episode 2. +[INFO] [1736286100.415964]: Saving video... +[WARN] [1736286101.043723]: Plan exhausted +[WARN] [1736286101.083732]: Plan exhausted +[WARN] [1736286101.125677]: Plan exhausted +[INFO] [1736286101.093618]: Video saved! +Starting episode 2 +[INFO] [1736286101.129830]: resetting environment +[INFO] [1736286101.152658]: cleared current plan +[INFO] [1736286101.152958]: moving to home +[INFO] [1736286104.555906]: reached home +[INFO] [1736286104.562178]: reset action history +[INFO] [1736286107.352538]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736286110.716276]: resetting environment +[INFO] [1736286110.729270]: cleared current plan +[INFO] [1736286110.729724]: moving to home +[INFO] [1736286111.732345]: reached home +[INFO] [1736286111.733429]: reset action history +[INFO] [1736286114.441725]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/episode_rosbags/episode_2_2025-01-07-16-41-54.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2187297.log +started roslaunch server http://158.130.50.37:44963/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2187866] +process[print_text-8]: started with pid [2187867] +process[rosbag_record-9]: started with pid [2187891] +started bagging! +For topic gripper_width: timestamp difference is 104886551 for nearest: 1736286123245530624 - target: 1736286123140644073 at idx 120 +For topic gripper_width: timestamp difference is 107547068 for nearest: 1736286124545512223 - target: 1736286124437965155 at idx 141 +For topic gripper_width: timestamp difference is 106895428 for nearest: 1736286134478886824 - target: 1736286134371991396 at idx 249 +[INFO] [1736286149.359496]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736286150.313573]: Stored episode 3. +[INFO] [1736286150.316314]: Saving video... +[WARN] [1736286150.693634]: Plan exhausted +[WARN] [1736286150.734602]: Plan exhausted +[WARN] [1736286150.775726]: Plan exhausted +[WARN] [1736286150.813630]: Plan exhausted +[WARN] [1736286150.853925]: Plan exhausted +[INFO] [1736286150.803364]: Video saved! +Starting episode 3 +[INFO] [1736286150.863345]: resetting environment +[INFO] [1736286150.886508]: cleared current plan +[INFO] [1736286150.888074]: moving to home +[INFO] [1736286153.892676]: reached home +[INFO] [1736286153.897894]: reset action history +[INFO] [1736286156.700223]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736286190.135751]: resetting environment +[INFO] [1736286190.175810]: cleared current plan +[INFO] [1736286190.178025]: moving to home +[INFO] [1736286191.181459]: reached home +[INFO] [1736286191.182217]: reset action history +[INFO] [1736286193.975663]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/episode_rosbags/episode_3_2025-01-07-16-43-13.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2187297.log +started roslaunch server http://158.130.50.37:36097/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2188052] +process[print_text-11]: started with pid [2188053] +process[rosbag_record-12]: started with pid [2188054] +started bagging! +For topic gripper_width: timestamp difference is -101218556 for nearest: 1736286195678940679 - target: 1736286195780159235 at idx 27 +For topic gripper_width: timestamp difference is 113827914 for nearest: 1736286220345530003 - target: 1736286220231702089 at idx 249 +For topic gripper_width: timestamp difference is 115452099 for nearest: 1736286221145529795 - target: 1736286221030077696 at idx 249 +[WARN] [1736286230.573903]: Plan exhausted +[WARN] [1736286230.613621]: Plan exhausted +[WARN] [1736286230.653697]: Plan exhausted +[INFO] [1736286229.223199]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736286229.973995]: Stored episode 4. +[INFO] [1736286229.978026]: Saving video... +[INFO] [1736286230.563666]: Video saved! +Starting episode 4 +[INFO] [1736286230.629431]: resetting environment +[INFO] [1736286230.663484]: cleared current plan +[INFO] [1736286230.663757]: moving to home +[INFO] [1736286234.175244]: reached home +[INFO] [1736286234.180196]: reset action history +[INFO] [1736286236.976359]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736286252.498630]: resetting environment +[INFO] [1736286252.547677]: cleared current plan +[INFO] [1736286252.552248]: moving to home +[INFO] [1736286253.562289]: reached home +[INFO] [1736286253.572218]: reset action history +[INFO] [1736286256.355802]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/73h7fwcu/episode_rosbags/episode_4_2025-01-07-16-44-16.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2187297.log +started roslaunch server http://158.130.50.37:41171/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2188194] +process[print_text-14]: started with pid [2188195] +process[rosbag_record-15]: started with pid [2188196] +started bagging! +For topic gripper_width: timestamp difference is -120287631 for nearest: 1736286269878874089 - target: 1736286269999161720 at idx 191 +For topic gripper_width: timestamp difference is -105615531 for nearest: 1736286282678821410 - target: 1736286282784436941 at idx 249 +[INFO] [1736286291.492491]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736286292.235655]: Stored episode 5. +[INFO] [1736286292.239938]: Saving video... +[WARN] [1736286292.837599]: Plan exhausted +[WARN] [1736286292.860514]: Plan exhausted +[WARN] [1736286292.893902]: Plan exhausted +[INFO] [1736286292.819429]: Video saved! +[INFO] [1736286292.886611]: resetting environment +[INFO] [1736286292.913476]: cleared current plan +[INFO] [1736286292.914095]: moving to home +[INFO] [1736286295.929808]: reached home +[INFO] [1736286295.933763]: reset action history +[INFO] [1736286298.750589]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/requirements.txt b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-metadata.json b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..f43749129fcd35fa590f2bdc75559ab1c382803d --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T21:39:50.553160", + "startedAt": "2025-01-07T21:39:50.204123", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3560.988625, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4496.073, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4495.306, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 687.2928199768066 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..aa8d98b356f120c77874f47967a6cf1e8f0ae564 --- /dev/null +++ b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "67a649dac0864a0b9776f767bc7576b313825aedd1fb1e5bd8232dcfb9a9c3be", "size": 1465999, "path": "media/videos/eval/0_eval_0_67a649dac0864a0b9776.mp4"}, "global_step": 665, "_timestamp": 1736286308.5551977, "_runtime": 318.3417308330536, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "cffb0391d794a1519dbaf82e22d4d4b11d19cd4cb117756b1dd14997b708132a", "size": 1495191, "path": "media/videos/eval/1_eval_1_cffb0391d794a1519dba.mp4"}, "eval/num_success": 0.0, "episode": 40.0, "eval/success_rate": 0.0, "eval/success_rate_lb_0.01": 0.0, "eval/success_rate_ub_0.01": 0.5363743901252747, "eval/success_rate_lb_0.025": 0.0, "eval/success_rate_ub_0.025": 0.4841596484184265, "eval/success_rate_lb_0.05": 0.0, "eval/success_rate_ub_0.05": 0.21277225017547607, "eval/success_rate_lb_0.1": 0.0, "eval/success_rate_ub_0.1": 0.167350634932518, "eval/2_eval": {"_type": "video-file", "sha256": "43ad98f5bb65a23f42a77ee3e7485ca244a84f2eca11d653428a694f8dbc7587", "size": 1523457, "path": "media/videos/eval/2_eval_2_43ad98f5bb65a23f42a7.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "5b62306ad8696d8b8c905da27e65c2d240f6e4906b7c2a1fcbe93a1da19fac61", "size": 1485499, "path": "media/videos/eval/3_eval_3_5b62306ad8696d8b8c90.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "66fc966ceba4d62103d365426065720163bd13fbbc1c8e542447c5b4dabe78dd", "size": 1502654, "path": "media/videos/eval/4_eval_4_66fc966ceba4d62103d3.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "e6508f0181aa84573747f0d380a2c6725005451ce9eb7fcdf77e1f39c90f7528", "size": 332, "artifact_path": "wandb-client-artifact://z318y79ay5g891wxj3poov74sexpy8py9qrxifamcm230wdj81f7ikpg9zb1jh3ua7smf7hjfp06s2ntrl3caf0zv2brd6s5xrk78dofq922b89zq9tr28r5hn4wg2br/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://wt4a4wpclyk32webglfx084lu2t19gmfmisd9365j1qnfntc7yj30k68vk0naiydnhr5zd71we9v60exxb8fz62mxl4677l62m9lb2bx2qv3tyly1qquw0m10kxpocid:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_e6508f0181aa84573747.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 25, "total_num_episodes": 40, "total_success_rate": 0.625, "total_success_rate_lb_0.01": 0.4539579749199444, "total_success_rate_ub_0.01": 0.7813661097917493, "total_success_rate_lb_0.025": 0.4635710865332028, "total_success_rate_ub_0.025": 0.7571975811801546, "total_success_rate_lb_0.05": 0.4847957044870268, "total_success_rate_ub_0.05": 0.7443524449575001, "total_success_rate_lb_0.1": 0.5340144000879443, "total_success_rate_ub_0.1": 0.7229229919166562, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "8532fe6ae6edcefe2d2bae63aaa26430eaaab95085b3bd2e29bf99158b42bd2a", "size": 507, "artifact_path": "wandb-client-artifact://8nh4ifnmbje8lxh1xgsebazmgkhnvqt731wjjcete99tf842jgqzrdjym60lh1vf9gvglkdfxxr55vf5zv595w5q0wnot2g7n75gxfywe5yijjm9mzpdjbv2zfa2kzw6/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://uy8430xdvinom7jsvirq9nz0t7iif6dqjnfwcf62bcbz5lfxan0eatvmnwo17l5snzm68rsiufeu1xpw34g828degozpctysk133s2zws2rj9fu3wwif9eo1khpqagn5:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_8532fe6ae6edcefe2d2b.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 322}, "eval/total_success_rate": 0.625, "eval/total_success_rate_lb_0.01": 0.4539579749107361, "eval/total_success_rate_ub_0.01": 0.7813661098480225, "eval/total_success_rate_lb_0.025": 0.46357110142707825, "eval/total_success_rate_ub_0.025": 0.7571975588798523, "eval/total_success_rate_lb_0.05": 0.4847957193851471, "eval/total_success_rate_ub_0.05": 0.7443524599075317, "eval/total_success_rate_lb_0.1": 0.5340144038200378, "eval/total_success_rate_ub_0.1": 0.7229229807853699} \ No newline at end of file diff --git a/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/run-73h7fwcu.wandb b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/run-73h7fwcu.wandb new file mode 100644 index 0000000000000000000000000000000000000000..585e9e5ffd8fca699ae6bb8132b30b30a0137aa9 Binary files /dev/null and b/73h7fwcu/wandb/run-20250107_163950-73h7fwcu/run-73h7fwcu.wandb differ diff --git a/bknjqqyi/.hydra/config.yaml b/bknjqqyi/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3 --- /dev/null +++ b/bknjqqyi/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1013_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/bknjqqyi/.hydra/hydra.yaml b/bknjqqyi/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7662af254630c7761a7dde2ce40de7752451fd94 --- /dev/null +++ b/bknjqqyi/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/bknjqqyi + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/bknjqqyi/.hydra/overrides.yaml b/bknjqqyi/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/bknjqqyi/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/bknjqqyi/eval_policy.log b/bknjqqyi/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..605b1a0c4c2b9e4ae77e4527ae226c06f8d06949 --- /dev/null +++ b/bknjqqyi/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-07 18:17:35,628][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 18:17:35,632][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-07 18:17:39,414][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) + diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23c4ab76eb8fac18019bec8ce3deec5eb8b6686d --- /dev/null +++ b/config.yaml @@ -0,0 +1,546 @@ +root_dir: /mnt/kostas-graid/datasets/extrinsic_contact_data +replay_buffer_size: 150000 +replay_buffer_num_workers: 2 +nstep: 3 +batch_size: 128 +seed: 1 +dataset_shuffle_seed: 0 +device: cuda +save_video: true +save_train_video: true +use_tb: true +use_wandb: true +wandb_run_id: '1013_1' +wandb_notes: 1013_1_restarted_11 +eval: false +true_action_history: false +train_pad_after: 4 +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: true +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: true +use_sdf_maps: true +use_normals_maps: true +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.2 +env_dtc_max_value: 0.4 +grasped_normals_mask_max_dtc_value: 0.2 +env_normals_mask_max_dtc_value: 0.4 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: true +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: small +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +baseline: false +train_demo_idxs_list_or_num: -1 +log_train_every_steps: 25 +name_of_expert_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +store_dataset_in_memory: false +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +include_groups_list: all +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +name_of_valid_demo: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_valid_demo}/demos.zarr +valid_demo_idxs_list_or_num: null +val_num_groups: 5 +load_bc: ${agent.load_checkpoint} +checkpoint_epoch_list: +- 99 +- 199 +- 299 +- 399 +- 499 +- 599 +- 699 +- 799 +- 899 +- 999 +- 1249 +- 1499 +- 1749 +- 1999 +- 2999 +- 3999 +- 4999 +- 5999 +- 6999 +- 7999 +- 8999 +- 9999 +snapshot_root_dir: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: true +save_last_snapshot: true +save_snapshot_when_done: true +top_k_checkpoints: 5 +save_snapshot_link_to_weights_dir: deprecated +bc_regularize: false +bc_weight_type: qfilter +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 1000000 + num_seed_frames: 0 + num_train_epochs: 15000 + validate_every_epochs: 250 + validate_diffusion_on_action_loss_every_epochs: 250 + train_eval_diffusion_on_action_loss_every_epochs: 250 + check_topk_every_epochs: 250 + save_snapshot_every_epochs: 1500 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 15000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 250 +validate_diffusion_on_action_loss_every_epochs: 250 +train_eval_diffusion_on_action_loss_every_epochs: 250 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: 250 +save_snapshot_every_epochs_diffusion: 1500 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 diff --git a/no798ka2/.hydra/config.yaml b/no798ka2/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3 --- /dev/null +++ b/no798ka2/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1013_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/no798ka2/.hydra/hydra.yaml b/no798ka2/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f32cb566a1ab7b48049a6a1188790d02854f9a26 --- /dev/null +++ b/no798ka2/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2 + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/no798ka2/.hydra/overrides.yaml b/no798ka2/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/no798ka2/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/no798ka2/episode_rosbags/aligned_depth_to_color_K.npy b/no798ka2/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/no798ka2/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/no798ka2/episode_rosbags/cam_tf_world.npy b/no798ka2/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/no798ka2/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/no798ka2/episode_rosbags/color_K.npy b/no798ka2/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/no798ka2/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/no798ka2/episode_rosbags/depth_K.npy b/no798ka2/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/no798ka2/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/no798ka2/eval_robot.log b/no798ka2/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..63a88b531244b9b080a211c1b83f377a3ec62266 --- /dev/null +++ b/no798ka2/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 18:23:04,315][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 18:23:04,318][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/no798ka2/eval_video/0_eval.mp4 b/no798ka2/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eb8f4604d06a940fad3801a204c6196f7119ebb --- /dev/null +++ b/no798ka2/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3c69d41167ae5a9086a17b4a2606e90c96c74aa1a8a6b32e56a5d151af825b7 +size 1417167 diff --git a/no798ka2/eval_video/1_eval.mp4 b/no798ka2/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a474638ab5f053f57303d3cca97ae320bf3a3a5b --- /dev/null +++ b/no798ka2/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddb2bcf1bbe47f8bb323ec9fe2265e13993d3dc3a5b539a328667e494dcd1176 +size 1424426 diff --git a/no798ka2/eval_video/2_eval.mp4 b/no798ka2/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a349e76d6b660b0da9359ccdae67138aaf934b7 --- /dev/null +++ b/no798ka2/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05d28ce3514e2b70c559deb00bf52ac7d13db1da0372a10ee394190011dbb3ee +size 1430708 diff --git a/no798ka2/eval_video/3_eval.mp4 b/no798ka2/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5bf7eacbda0a51b19d296d10e3c3b7968880d322 --- /dev/null +++ b/no798ka2/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a5d4eef35ecdfe1339908dc4227c2cd69015e75d84653341550b8667f1b6adf +size 1435941 diff --git a/no798ka2/eval_video/4_eval.mp4 b/no798ka2/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2d64194d8f9a105e931357099a6fc71a5de08083 --- /dev/null +++ b/no798ka2/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8bdad0cca02a9e0014215ae4de330566b89620ab36208ea2937d17365eec38f +size 1405906 diff --git a/no798ka2/tb/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 b/no798ka2/tb/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 new file mode 100644 index 0000000000000000000000000000000000000000..d6c70587bbdfe654b465f74a1d3443588518f6c4 --- /dev/null +++ b/no798ka2/tb/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:334324d840f3b0ae5b7c5632bdc769b1287e5fc92af5e5a5a74c6dfedbcd2347 +size 6539 diff --git a/no798ka2/wandb/debug-internal.log b/no798ka2/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bfe14a560fda3652eac32193af9e7455d179525d --- /dev/null +++ b/no798ka2/wandb/debug-internal.log @@ -0,0 +1,801 @@ +2025-01-07 18:23:10,073 INFO StreamThr :2211704 [internal.py:wandb_internal():85] W&B internal server running at pid: 2211704, started at: 2025-01-07 18:23:10.072732 +2025-01-07 18:23:10,074 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:23:10,091 INFO WriterThread:2211704 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/run-no798ka2.wandb +2025-01-07 18:23:10,092 DEBUG SenderThread:2211704 [sender.py:send():379] send: header +2025-01-07 18:23:10,098 DEBUG SenderThread:2211704 [sender.py:send():379] send: run +2025-01-07 18:23:10,371 INFO SenderThread:2211704 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files +2025-01-07 18:23:10,371 INFO SenderThread:2211704 [sender.py:_start_run_threads():1188] run started: no798ka2 with start time 1736292190.073567 +2025-01-07 18:23:10,390 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:23:10,391 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:23:10,469 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:23:10,474 DEBUG HandlerThread:2211704 [system_info.py:__init__():26] System info init +2025-01-07 18:23:10,474 DEBUG HandlerThread:2211704 [system_info.py:__init__():41] System info init done +2025-01-07 18:23:10,474 INFO HandlerThread:2211704 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:23:10,474 INFO SystemMonitor:2211704 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:23:10,474 INFO HandlerThread:2211704 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:23:10,475 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:23:10,475 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:23:10,476 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:23:10,477 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:23:10,477 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:23:10,487 DEBUG HandlerThread:2211704 [system_info.py:probe():152] Probing system +2025-01-07 18:23:10,488 DEBUG HandlerThread:2211704 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:23:10,493 DEBUG HandlerThread:2211704 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:23:10,493 DEBUG HandlerThread:2211704 [system_info.py:probe():200] Probing system done +2025-01-07 18:23:10,493 DEBUG HandlerThread:2211704 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:23:10.487545', 'startedAt': '2025-01-07T23:23:10.063101', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.6589999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.277, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 838.028434753418}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:23:10,493 INFO HandlerThread:2211704 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:23:10,493 INFO HandlerThread:2211704 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:23:10,493 DEBUG HandlerThread:2211704 [system_info.py:_save_code():44] Saving code +2025-01-07 18:23:10,495 DEBUG HandlerThread:2211704 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:23:10,495 DEBUG HandlerThread:2211704 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:23:10,556 DEBUG HandlerThread:2211704 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:23:10,557 INFO HandlerThread:2211704 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:23:10,560 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:23:10,560 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:23:10,560 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:23:10,561 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 18:23:10,807 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/tv42cjfj-wandb-metadata.json +2025-01-07 18:23:10,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:23:10,813 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:23:10,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:23:10,813 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:23:10,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:10,816 INFO HandlerThread:2211704 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/tb" + save: true +} +control { + relay_id: "d6d220aa51274965a20635b41aeddf86" +} +_info { + stream_id: "no798ka2" +} + +2025-01-07 18:23:10,817 DEBUG HandlerThread:2211704 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:23:10,818 INFO wandb-upload_2:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/htub1ma4-diff.patch +2025-01-07 18:23:10,842 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/n93w829l-code/FISH/eval_robot.py +2025-01-07 18:23:10,916 DEBUG SenderThread:2211704 [sender.py:send():379] send: telemetry +2025-01-07 18:23:10,916 DEBUG SenderThread:2211704 [sender.py:send():379] send: tbrecord +2025-01-07 18:23:10,917 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:23:10,917 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 with policy live +2025-01-07 18:23:11,144 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/2j7sy6aw-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:23:11,372 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-metadata.json +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/code/FISH/eval_robot.py +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/requirements.txt +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/diff.patch +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/code/FISH +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/code +2025-01-07 18:23:11,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:12,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:13,372 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:13,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:14,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:15,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:15,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:16,495 DEBUG SenderThread:2211704 [sender.py:send():379] send: config +2025-01-07 18:23:16,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:17,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:18,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:19,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:19,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:20,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:20,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:21,374 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:21,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:22,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:23,375 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:23,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:24,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:25,375 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:25,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:23:25,813 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:23:25,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:25,925 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:26,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:27,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:28,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:29,376 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:29,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:30,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:31,377 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:31,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:31,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:32,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:33,378 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:33,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:34,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:35,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:36,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:36,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:37,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:38,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:39,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:40,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:23:40,814 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:23:40,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:41,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:41,948 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:42,380 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml +2025-01-07 18:23:42,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:43,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:44,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:45,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:46,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:47,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:47,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:48,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:49,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:50,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:51,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:52,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:52,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:53,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:54,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:55,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:23:55,814 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:23:55,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:56,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:57,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:57,936 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:58,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:59,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:00,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:01,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:02,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:03,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:03,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:04,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:05,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:06,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:07,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:07,961 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:24:07,962 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:24:07,962 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 with policy now +2025-01-07 18:24:07,962 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:24:07,962 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:24:07,963 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:24:08,233 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/udm3crnc-media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:24:08,687 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/9p7ujrt5-media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 +2025-01-07 18:24:08,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:08,964 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:09,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:09,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:10,477 DEBUG SystemMonitor:2211704 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:24:10,478 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:24:10,819 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:24:10,819 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:24:10,861 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:11,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:12,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:13,389 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:13,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:14,354 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:14,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:15,389 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:15,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:16,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:17,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:18,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:19,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:19,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:20,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:21,391 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:24:21,657 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/pie1o1m1-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:24:21,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:22,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:23,392 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:23,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:24,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:24,950 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:25,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:24:25,815 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:24:25,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:26,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:27,393 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:27,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:28,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:29,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:30,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:30,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:31,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:32,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:33,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:34,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:35,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:35,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:36,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:37,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:38,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:39,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:40,478 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:24:40,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:24:40,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:24:40,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:40,943 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:41,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:42,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:43,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:44,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:45,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:46,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:46,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:47,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:48,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:49,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:50,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:51,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:51,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:52,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:53,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:54,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:55,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:24:55,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:24:55,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:56,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:56,896 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:57,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:58,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:59,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:00,657 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:25:00,719 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:25:00,719 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 with policy now +2025-01-07 18:25:00,720 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:25:00,720 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:25:00,721 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:25:00,834 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:00,993 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/0hpy56cy-media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 +2025-01-07 18:25:01,405 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:25:01,405 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:01,405 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 +2025-01-07 18:25:01,406 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:25:01,714 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/k1oiv7il-media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 +2025-01-07 18:25:01,832 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:02,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:02,838 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:03,406 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:03,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:04,836 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:05,406 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:05,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:06,837 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:07,835 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:08,217 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:08,836 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:09,408 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:09,836 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:10,479 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:25:10,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:25:10,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:25:10,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:11,837 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:12,837 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:13,839 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:13,874 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:14,838 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:15,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:16,838 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:17,839 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:18,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:19,357 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:19,411 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:25:19,839 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:20,242 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:25:20,243 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:25:20,244 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:25:20,411 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:25:20,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:21,412 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:21,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:22,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:23,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:24,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:25,341 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:25,413 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:25,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:25:25,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:25:25,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:26,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:27,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:28,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:29,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:30,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:30,844 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:31,844 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:32,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:33,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:34,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:35,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:35,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:36,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:37,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:38,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:39,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:40,479 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:25:40,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:25:40,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:25:40,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:40,967 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:41,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:42,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:43,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:44,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:45,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:45,997 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:46,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:47,420 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:47,847 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:48,847 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:49,421 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:49,847 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:50,847 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:51,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:51,848 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:52,849 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:53,848 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:54,848 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:55,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:25:55,817 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:25:55,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:56,848 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:56,946 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:57,849 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:58,860 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:58,981 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:25:58,981 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:25:58,981 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 with policy now +2025-01-07 18:25:58,982 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:25:58,982 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:25:58,999 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:25:59,256 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/o4dox3pn-media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 +2025-01-07 18:25:59,424 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:25:59,425 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:59,425 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 +2025-01-07 18:25:59,425 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:25:59,708 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/1v94weyc-media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 +2025-01-07 18:25:59,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:00,853 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:01,425 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:01,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:02,705 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:02,861 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:03,860 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:04,866 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:05,427 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:05,861 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:06,863 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:07,427 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:07,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:07,870 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:08,867 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:09,873 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:10,480 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:26:10,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:26:10,823 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:26:10,874 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:11,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:12,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:12,973 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:13,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:14,088 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:26:14,088 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:26:14,089 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:26:14,430 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:26:14,430 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:26:14,720 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/vrrxb54c-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:26:14,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:15,430 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:15,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:16,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:17,431 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:17,874 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:17,976 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:18,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:19,431 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:19,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:20,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:21,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:22,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:23,830 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:23,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:24,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:25,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:26:25,820 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:26:25,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:26,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:27,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:28,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:28,913 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:29,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:30,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:31,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:32,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:33,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:34,689 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:34,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:35,437 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:35,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:36,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:37,438 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:37,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:38,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:39,831 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:39,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:40,480 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:26:40,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:26:40,820 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:26:40,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:41,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:42,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:43,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:44,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:44,890 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:45,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:46,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:47,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:48,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:49,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:50,831 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:50,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:51,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:52,885 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:53,443 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:53,882 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:54,016 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:26:54,016 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 with policy now +2025-01-07 18:26:54,020 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:26:54,020 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:26:54,020 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:26:54,026 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:26:54,279 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/xjbh0wj3-media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 +2025-01-07 18:26:54,443 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:26:54,444 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 +2025-01-07 18:26:54,444 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:26:54,745 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/nob6d7t0-media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 +2025-01-07 18:26:54,884 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:55,444 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:55,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:26:55,833 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:26:55,890 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:55,910 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:56,444 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:56,885 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:57,885 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:58,892 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:59,445 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:59,891 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:00,882 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:01,446 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:01,832 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:01,895 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:02,889 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:03,892 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:04,889 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:05,895 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:06,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:06,896 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:07,897 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:08,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:09,155 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:27:09,155 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:27:09,156 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:27:09,449 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:27:09,449 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:27:09,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:10,481 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:27:10,835 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:27:10,835 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:27:10,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:11,449 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:11,903 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:11,951 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:12,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:13,897 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:14,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:15,451 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:15,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:16,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:17,259 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:17,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:18,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:19,452 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:19,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:20,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:21,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:22,832 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:22,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:23,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:24,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:25,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:27:25,823 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:27:25,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:26,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:27,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:27,931 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:28,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:29,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:30,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:31,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:32,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:33,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:33,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:34,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:35,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:36,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:37,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:38,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:39,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:39,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:40,481 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:27:40,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:27:40,823 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:27:40,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:41,903 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:42,902 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:43,902 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:44,902 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:44,959 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:45,902 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:46,903 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:47,903 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:48,906 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:48,951 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:27:48,951 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 with policy now +2025-01-07 18:27:48,993 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:27:48,994 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:27:48,994 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:27:48,995 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:27:49,235 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/ytpxu9di-media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 +2025-01-07 18:27:49,462 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:27:49,463 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:49,463 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 +2025-01-07 18:27:49,463 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:27:49,743 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/8uwvw4rl-media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 +2025-01-07 18:27:49,911 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:49,996 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:50,915 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:51,463 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:51,906 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:52,930 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:53,464 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:53,917 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:54,914 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:55,464 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:55,499 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:55,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:27:55,829 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:27:55,923 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:56,921 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:57,465 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:57,927 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:58,925 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:59,216 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:27:59,216 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:27:59,217 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:27:59,466 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:27:59,466 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:27:59,518 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:27:59,518 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:27:59,674 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:27:59,727 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/xfeu384j-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:27:59,921 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:00,007 INFO wandb-upload_0:2211704 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmqqm3ukd +2025-01-07 18:28:00,076 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:28:00,087 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:28:00,402 INFO SenderThread:2211704 [sender.py:send_request_log_artifact():1525] logged artifact run-no798ka2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjExMzE3Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc1NTkyOA==', 'latestArtifact': None}} +2025-01-07 18:28:00,402 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:28:00,402 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 18:28:00,402 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:28:00,402 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,403 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,403 DEBUG SenderThread:2211704 [sender.py:send():379] send: telemetry +2025-01-07 18:28:00,403 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,407 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,407 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,408 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,408 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,408 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,408 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,409 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,409 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,409 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,409 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,410 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,410 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,410 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,410 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,410 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,411 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,411 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,411 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,411 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,412 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,412 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,412 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:28:00,466 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:00,466 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table +2025-01-07 18:28:00,585 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/h7qe27r4-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 18:28:00,676 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/6hy7j2ag-media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json +2025-01-07 18:28:00,774 INFO wandb-upload_2:2211704 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp00tumeh3 +2025-01-07 18:28:00,846 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/89tlbp6u-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 18:28:00,922 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:01,166 INFO SenderThread:2211704 [sender.py:send_request_log_artifact():1525] logged artifact run-no798ka2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjExMzIwMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc1NTkzMA==', 'latestArtifact': None}} +2025-01-07 18:28:01,170 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:01,275 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:28:01,275 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json with policy now +2025-01-07 18:28:01,275 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:28:01,275 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:01,275 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:01,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:01,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml +2025-01-07 18:28:01,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:28:01,531 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/175n4i24-media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json +2025-01-07 18:28:01,922 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:02,505 DEBUG SenderThread:2211704 [sender.py:send():379] send: exit +2025-01-07 18:28:02,505 INFO SenderThread:2211704 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:28:02,506 INFO SenderThread:2211704 [sender.py:send_exit():588] handling runtime: 292 +2025-01-07 18:28:02,506 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:02,506 INFO SenderThread:2211704 [sender.py:send_exit():594] send defer +2025-01-07 18:28:02,506 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,506 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:28:02,506 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:02,506 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:28:02,507 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:28:02,507 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,507 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:28:02,507 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:02,507 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:28:02,507 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:28:02,507 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,507 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:28:02,507 INFO HandlerThread:2211704 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:28:02,507 DEBUG SystemMonitor:2211704 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:28:02,507 DEBUG SystemMonitor:2211704 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:28:02,508 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:28:02,508 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:28:02,559 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:28:02,559 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:28:02,559 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:28:02,560 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:02,560 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:28:02,560 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:28:02,560 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:28:02,560 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,560 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:28:02,561 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:02,561 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:28:02,561 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:28:02,561 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,561 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:28:03,468 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:03,468 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:28:09,222 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:09,222 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:09,222 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:28:09,224 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:09,224 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:28:09,224 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:28:09,224 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:09,224 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:28:09,224 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:09,224 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:09,224 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:28:09,225 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:09,225 DEBUG SenderThread:2211704 [sender.py:send():379] send: summary +2025-01-07 18:28:09,226 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:09,226 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:09,226 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:28:09,226 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:28:09,226 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:09,226 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:28:09,226 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:09,226 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:28:09,227 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:28:09,227 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:09,227 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:09,227 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:28:09,227 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:09,227 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:28:09,470 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:09,507 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:12,037 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:28:12,037 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:12,037 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,037 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:28:12,038 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,038 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:28:12,038 INFO SenderThread:2211704 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:28:12,040 INFO SenderThread:2211704 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:28:12,042 INFO SenderThread:2211704 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:28:12,053 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:28:12,054 DEBUG SenderThread:2211704 [sender.py:send():379] send: artifact +2025-01-07 18:28:12,056 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,056 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:28:12,238 INFO SenderThread:2211704 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:28:12,238 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,238 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:28:12,238 INFO SenderThread:2211704 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:28:12,471 INFO SenderThread:2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:28:12,471 INFO SenderThread:2211704 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files +2025-01-07 18:28:12,471 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml config.yaml +2025-01-07 18:28:12,472 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json wandb-summary.json +2025-01-07 18:28:12,472 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/requirements.txt requirements.txt +2025-01-07 18:28:12,472 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:28:12,472 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:28:12,476 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log output.log +2025-01-07 18:28:12,476 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/diff.patch diff.patch +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 18:28:12,478 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json +2025-01-07 18:28:12,478 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:28:12,480 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,481 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:28:12,481 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,481 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:28:12,481 INFO SenderThread:2211704 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:28:12,508 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:12,508 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:12,655 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/requirements.txt +2025-01-07 18:28:12,661 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml +2025-01-07 18:28:12,672 INFO wandb-upload_2:2211704 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:12,688 INFO wandb-upload_4:2211704 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:28:12,756 INFO wandb-upload_3:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/9acx95ix-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:28:12,957 INFO Thread-11 (_thread_body):2211704 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:28:12,957 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,957 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:28:12,957 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,957 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:28:12,957 INFO SenderThread:2211704 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:28:12,959 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:28:12,959 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,959 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:28:12,959 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,959 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:28:12,959 INFO SenderThread:2211704 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:28:13,160 INFO SenderThread:2211704 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:28:13,160 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:28:13,160 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:13,160 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:28:13,160 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:13,160 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:28:13,160 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:28:13,160 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:13,160 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:28:13,161 DEBUG SenderThread:2211704 [sender.py:send():379] send: final +2025-01-07 18:28:13,161 DEBUG SenderThread:2211704 [sender.py:send():379] send: footer +2025-01-07 18:28:13,161 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:13,161 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:28:13,161 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:13,161 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:28:13,161 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:13,162 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:13,162 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:28:13,164 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:28:13,164 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:28:13,165 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:13,210 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:13,210 INFO MainThread:2211704 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:28:13,211 INFO MainThread:2211704 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:28:13,211 INFO MainThread:2211704 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:28:13,211 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:28:13,211 INFO HandlerThread:2211704 [handler.py:finish():882] shutting down handler +2025-01-07 18:28:14,162 INFO WriterThread:2211704 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/run-no798ka2.wandb +2025-01-07 18:28:14,210 INFO SenderThread:2211704 [sender.py:finish():1615] shutting down sender +2025-01-07 18:28:14,210 INFO SenderThread:2211704 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:28:14,210 INFO SenderThread:2211704 [file_pusher.py:join():175] waiting for file pusher diff --git a/no798ka2/wandb/debug.log b/no798ka2/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..825c2fc5306ed34c493b4564d261c06a260bbddb --- /dev/null +++ b/no798ka2/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Configure stats pid to 2211530 +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/settings +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug.log +2025-01-07 18:23:10,067 INFO MainThread:2211530 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug-internal.log +2025-01-07 18:23:10,067 INFO MainThread:2211530 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:23:10,067 INFO MainThread:2211530 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2'} +2025-01-07 18:23:10,068 INFO MainThread:2211530 [wandb_init.py:init():619] starting backend +2025-01-07 18:23:10,068 INFO MainThread:2211530 [wandb_init.py:init():623] setting up manager +2025-01-07 18:23:10,071 INFO MainThread:2211530 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:23:10,073 INFO MainThread:2211530 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:23:10,090 INFO MainThread:2211530 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:23:10,096 INFO MainThread:2211530 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:23:10,390 INFO MainThread:2211530 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:23:10,460 INFO MainThread:2211530 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:23:10,460 INFO MainThread:2211530 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:23:10,813 INFO MainThread:2211530 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:23:10,813 INFO MainThread:2211530 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:23:10,813 INFO MainThread:2211530 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:23:10,813 INFO MainThread:2211530 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:23:10,816 INFO MainThread:2211530 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:23:10,816 INFO MainThread:2211530 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/tb, True +2025-01-07 18:23:16,494 INFO MainThread:2211530 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:28:14,212 WARNING MsgRouterThr:2211530 [router.py:message_loop():77] message_loop has been closed diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/code/FISH/eval_robot.py b/no798ka2/wandb/run-20250107_182310-no798ka2/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml b/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13daf31154414f7ef4f0ab7ada8c191a2f09f38b --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1013_1' +wandb_notes: + desc: null + value: 1013_1_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2 +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736292190 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: lib +left_book_slot: + desc: null + value: twodim diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/diff.patch b/no798ka2/wandb/run-20250107_182310-no798ka2/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..8adda0290b36965f42f4119e05fd13ea9507158f --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/diff.patch @@ -0,0 +1,15 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..1c0be7a 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -139,8 +139,8 @@ load_checkpoint: ${agent.load_checkpoint} + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' +-# wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..eb10de88a713376e1c7301b69ab12adffa731488 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[1.0, 0.7501049041247838, 1, 0.6051223873881857, 1, 0.6927025019737123, 1, 0.6332908421488255, 1]]} \ No newline at end of file diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json new file mode 100644 index 0000000000000000000000000000000000000000..3093d544f362f78ab823613ea3d901cad65ae860 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[34, 50, 0.68, 0.5274859368746145, 0.8127126097053713, 0.5377310514374611, 0.7924793362032534, 0.5571022108083008, 0.78122860187628, 0.599585097263208, 0.7629664819161672]]} \ No newline at end of file diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0eb8f4604d06a940fad3801a204c6196f7119ebb --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c3c69d41167ae5a9086a17b4a2606e90c96c74aa1a8a6b32e56a5d151af825b7 +size 1417167 diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..a474638ab5f053f57303d3cca97ae320bf3a3a5b --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ddb2bcf1bbe47f8bb323ec9fe2265e13993d3dc3a5b539a328667e494dcd1176 +size 1424426 diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5a349e76d6b660b0da9359ccdae67138aaf934b7 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:05d28ce3514e2b70c559deb00bf52ac7d13db1da0372a10ee394190011dbb3ee +size 1430708 diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..5bf7eacbda0a51b19d296d10e3c3b7968880d322 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a5d4eef35ecdfe1339908dc4227c2cd69015e75d84653341550b8667f1b6adf +size 1435941 diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..2d64194d8f9a105e931357099a6fc71a5de08083 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8bdad0cca02a9e0014215ae4de330566b89620ab36208ea2937d17365eec38f +size 1405906 diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log b/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..8b3d17f4fa11f6b881481e62ffbef649edb979a4 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log @@ -0,0 +1,270 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736292196.495169]: resetting environment +[INFO] [1736292196.499691]: cleared current plan +[INFO] [1736292196.500542]: moving to home +[INFO] [1736292197.502330]: reached home +[INFO] [1736292197.503625]: reset action history +[INFO] [1736292200.213979]: environment reset +Starting episode 0 +[INFO] [1736292200.218888]: resetting environment +[INFO] [1736292200.226277]: cleared current plan +[INFO] [1736292200.227123]: moving to home +[INFO] [1736292201.229195]: reached home +[INFO] [1736292201.230451]: reset action history +[INFO] [1736292203.939976]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736292208.350751]: resetting environment +[INFO] [1736292208.355079]: cleared current plan +[INFO] [1736292208.356422]: moving to home +[INFO] [1736292209.359691]: reached home +[INFO] [1736292209.360991]: reset action history +[INFO] [1736292212.070130]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/episode_rosbags/episode_0_2025-01-07-18-23-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2211530.log +started roslaunch server http://158.130.50.37:34943/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2211821] +process[print_text-2]: started with pid [2211845] +process[rosbag_record-3]: started with pid [2211846] +started bagging! +[INFO] [1736292246.928477]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736292247.729589]: Stored episode 1. +[INFO] [1736292247.730459]: Saving video... +[INFO] [1736292247.919985]: Video saved! +Starting episode 1 +[INFO] [1736292247.931495]: resetting environment +[INFO] [1736292247.936675]: cleared current plan +[INFO] [1736292247.937318]: moving to home +[INFO] [1736292251.640011]: reached home +[INFO] [1736292251.640487]: reset action history +[INFO] [1736292254.353766]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292261.236300]: resetting environment +[INFO] [1736292261.241338]: cleared current plan +[INFO] [1736292261.241520]: moving to home +[INFO] [1736292262.243232]: reached home +[INFO] [1736292262.243856]: reset action history +[INFO] [1736292264.950443]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/episode_rosbags/episode_1_2025-01-07-18-24-24.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2211530.log +started roslaunch server http://158.130.50.37:43279/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2211982] +process[print_text-5]: started with pid [2211983] +process[rosbag_record-6]: started with pid [2211984] +started bagging! +[INFO] [1736292299.665678]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736292300.396321]: Stored episode 2. +[INFO] [1736292300.396531]: Saving video... +[INFO] [1736292300.607479]: Video saved! +Starting episode 2 +[INFO] [1736292300.659058]: resetting environment +[INFO] [1736292300.690942]: cleared current plan +[INFO] [1736292300.697195]: moving to home +[INFO] [1736292304.404563]: reached home +[INFO] [1736292304.406707]: reset action history +[INFO] [1736292307.175287]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292319.375905]: resetting environment +[INFO] [1736292319.387417]: cleared current plan +[INFO] [1736292319.387765]: moving to home +[INFO] [1736292320.389236]: reached home +[INFO] [1736292320.389680]: reset action history +[INFO] [1736292323.097774]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/episode_rosbags/episode_2_2025-01-07-18-25-23.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2211530.log +started roslaunch server http://158.130.50.37:38929/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2212124] +process[print_text-8]: started with pid [2212125] +process[rosbag_record-9]: started with pid [2212126] +started bagging! +For topic gripper_width: timestamp difference is 101426661 for nearest: 1736292345945469201 - target: 1736292345844042540 at idx 249 +For topic gripper_width: timestamp difference is 101495459 for nearest: 1736292348278798535 - target: 1736292348177303076 at idx 249 +[INFO] [1736292357.895147]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit +[pub_text-7] killing on exit +[INFO] [1736292358.659924]: Stored episode 3. +[INFO] [1736292358.663943]: Saving video... +[INFO] [1736292358.936617]: Video saved! +Starting episode 3 +[INFO] [1736292358.972785]: resetting environment +[INFO] [1736292358.996443]: cleared current plan +[INFO] [1736292358.996716]: moving to home +[INFO] [1736292362.704322]: reached home +[INFO] [1736292362.709594]: reset action history +[INFO] [1736292365.663243]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292373.984178]: resetting environment +[INFO] [1736292374.003404]: cleared current plan +[INFO] [1736292374.003691]: moving to home +[INFO] [1736292375.007373]: reached home +[INFO] [1736292375.012578]: reset action history +[INFO] [1736292377.790319]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/episode_rosbags/episode_3_2025-01-07-18-26-17.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2211530.log +started roslaunch server http://158.130.50.37:43343/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2212278] +process[print_text-11]: started with pid [2212279] +process[rosbag_record-12]: started with pid [2212303] +started bagging! +For topic gripper_width: timestamp difference is 102498339 for nearest: 1736292393645467566 - target: 1736292393542969227 at idx 206 +For topic gripper_width: timestamp difference is 103975581 for nearest: 1736292395445468234 - target: 1736292395341492653 at idx 234 +[INFO] [1736292412.598625]: Storing episode... +[rosbag_record-12] killing on exit +[pub_text-10] killing on exit +[print_text-11] killing on exit +[INFO] [1736292413.389523]: Stored episode 4. +[INFO] [1736292413.394647]: Saving video... +[INFO] [1736292413.971799]: Video saved! +Starting episode 4 +[INFO] [1736292414.025232]: resetting environment +[INFO] [1736292414.064359]: cleared current plan +[INFO] [1736292414.066583]: moving to home +[WARN] [1736292413.960611]: Plan exhausted +[WARN] [1736292414.001018]: Plan exhausted +[WARN] [1736292414.041668]: Plan exhausted +[INFO] [1736292417.775555]: reached home +[INFO] [1736292417.776534]: reset action history +[INFO] [1736292420.580088]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736292429.077051]: resetting environment +[INFO] [1736292429.118027]: cleared current plan +[INFO] [1736292429.118260]: moving to home +[INFO] [1736292430.122068]: reached home +[INFO] [1736292430.127519]: reset action history +[INFO] [1736292432.922760]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/episode_rosbags/episode_4_2025-01-07-18-27-12.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2211530.log +started roslaunch server http://158.130.50.37:37301/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2212433] +process[print_text-14]: started with pid [2212434] +process[rosbag_record-15]: started with pid [2212458] +started bagging! +For topic gripper_width: timestamp difference is 101978738 for nearest: 1736292434445466954 - target: 1736292434343488216 at idx 24 +For topic gripper_width: timestamp difference is 104284257 for nearest: 1736292437245466680 - target: 1736292437141182423 at idx 64 +[INFO] [1736292467.679946]: Storing episode... +[print_text-14] killing on exit +[pub_text-13] killing on exit[rosbag_record-15] killing on exit +[INFO] [1736292468.519968]: Stored episode 5. +[INFO] [1736292468.523427]: Saving video... +[INFO] [1736292468.894835]: Video saved! +[INFO] [1736292468.952558]: resetting environment +[INFO] [1736292468.985623]: cleared current plan +[INFO] [1736292468.985887]: moving to home +[INFO] [1736292472.689882]: reached home +[INFO] [1736292472.698284]: reset action history +[INFO] [1736292475.498073]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/requirements.txt b/no798ka2/wandb/run-20250107_182310-no798ka2/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-metadata.json b/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e34912cc8ca82d4045a59f4d59a0929d764dd6f7 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-07T23:23:10.487545", + "startedAt": "2025-01-07T23:23:10.063101", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3561.6589999999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.277, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 838.028434753418 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json b/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2d85b353d8551715afc8b2777ab77ec67c4bd219 --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "c3c69d41167ae5a9086a17b4a2606e90c96c74aa1a8a6b32e56a5d151af825b7", "size": 1417167, "path": "media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4"}, "global_step": 665, "_timestamp": 1736292480.00606, "_runtime": 289.9324929714203, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "ddb2bcf1bbe47f8bb323ec9fe2265e13993d3dc3a5b539a328667e494dcd1176", "size": 1424426, "path": "media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4"}, "eval/num_success": 5.0, "episode": 50.0, "eval/success_rate": 1.0, "eval/success_rate_lb_0.01": 0.7501049041748047, "eval/success_rate_ub_0.01": 1.0, "eval/success_rate_lb_0.025": 0.6051223874092102, "eval/success_rate_ub_0.025": 1.0, "eval/success_rate_lb_0.05": 0.6927024722099304, "eval/success_rate_ub_0.05": 1.0, "eval/success_rate_lb_0.1": 0.6332908272743225, "eval/success_rate_ub_0.1": 1.0, "eval/2_eval": {"_type": "video-file", "sha256": "05d28ce3514e2b70c559deb00bf52ac7d13db1da0372a10ee394190011dbb3ee", "size": 1430708, "path": "media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "3a5d4eef35ecdfe1339908dc4227c2cd69015e75d84653341550b8667f1b6adf", "size": 1435941, "path": "media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "e8bdad0cca02a9e0014215ae4de330566b89620ab36208ea2937d17365eec38f", "size": 1405906, "path": "media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "d8821c50291f0fbf979f12eb7aaa0b647dfd15fe05387335a9663efc94f7fc3c", "size": 330, "artifact_path": "wandb-client-artifact://dvve509nnlsbzcc9ntunow4obif62s6cczsxq88qxxtpiutk6jyu9i8rvgzdn19qvh7o8bv2r77730ui89ty1t00tzh3roucxa387dun9l6hap3awdkqoh3m007lbypz/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://fzaol1mfzftsol4q6u8o0czbfc7c1p133scmidpq3td7hoaglhmw7t8ctlizm6xk99wx2u2ea4au4fzfnthrr7msuk6rp9xwv4hoc75ygy2bkel7jomuhqr7o5417laa:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 34, "total_num_episodes": 50, "total_success_rate": 0.68, "total_success_rate_lb_0.01": 0.5274859368746145, "total_success_rate_ub_0.01": 0.8127126097053713, "total_success_rate_lb_0.025": 0.5377310514374611, "total_success_rate_ub_0.025": 0.7924793362032534, "total_success_rate_lb_0.05": 0.5571022108083008, "total_success_rate_ub_0.05": 0.78122860187628, "total_success_rate_lb_0.1": 0.599585097263208, "total_success_rate_ub_0.1": 0.7629664819161672, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "ac8537acf8bec14e3652e759fd9a6b82cf1831d5feb9106347d6d8b35cdbdd09", "size": 503, "artifact_path": "wandb-client-artifact://3wzjg8pyt8i7xavvg9kvyq7ocb2st73cnlgoevk29zd9yxlowzt6x7gvw64x8wcrj6nkrnnpur5ax69q62l0dpkmlss4o6ehhnj5s8z7oup0b8vnnhrdmqbfdqv4zeex/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://m67am93cfhjnfg9urgpdzx00rjfukp6ba2gwfwsmdslwnioe28rfu817youa9vl2nj3pxeelpsdkbg38k4l3i1949d99jn7pew1xqq76qph5b0jccd75uzolmbvuik2m:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 292}, "eval/total_success_rate": 0.6800000071525574, "eval/total_success_rate_lb_0.01": 0.5274859070777893, "eval/total_success_rate_ub_0.01": 0.8127126097679138, "eval/total_success_rate_lb_0.025": 0.5377310514450073, "eval/total_success_rate_ub_0.025": 0.7924793362617493, "eval/total_success_rate_lb_0.05": 0.5571022033691406, "eval/total_success_rate_ub_0.05": 0.7812286019325256, "eval/total_success_rate_lb_0.1": 0.5995851159095764, "eval/total_success_rate_ub_0.1": 0.7629664540290833} \ No newline at end of file diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug-internal.log b/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..bfe14a560fda3652eac32193af9e7455d179525d --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug-internal.log @@ -0,0 +1,801 @@ +2025-01-07 18:23:10,073 INFO StreamThr :2211704 [internal.py:wandb_internal():85] W&B internal server running at pid: 2211704, started at: 2025-01-07 18:23:10.072732 +2025-01-07 18:23:10,074 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status +2025-01-07 18:23:10,091 INFO WriterThread:2211704 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/run-no798ka2.wandb +2025-01-07 18:23:10,092 DEBUG SenderThread:2211704 [sender.py:send():379] send: header +2025-01-07 18:23:10,098 DEBUG SenderThread:2211704 [sender.py:send():379] send: run +2025-01-07 18:23:10,371 INFO SenderThread:2211704 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files +2025-01-07 18:23:10,371 INFO SenderThread:2211704 [sender.py:_start_run_threads():1188] run started: no798ka2 with start time 1736292190.073567 +2025-01-07 18:23:10,390 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 18:23:10,391 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: check_version +2025-01-07 18:23:10,469 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 18:23:10,474 DEBUG HandlerThread:2211704 [system_info.py:__init__():26] System info init +2025-01-07 18:23:10,474 DEBUG HandlerThread:2211704 [system_info.py:__init__():41] System info init done +2025-01-07 18:23:10,474 INFO HandlerThread:2211704 [system_monitor.py:start():194] Starting system monitor +2025-01-07 18:23:10,474 INFO SystemMonitor:2211704 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 18:23:10,474 INFO HandlerThread:2211704 [system_monitor.py:probe():214] Collecting system info +2025-01-07 18:23:10,475 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 18:23:10,475 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started disk monitoring +2025-01-07 18:23:10,476 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 18:23:10,477 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started memory monitoring +2025-01-07 18:23:10,477 INFO SystemMonitor:2211704 [interfaces.py:start():188] Started network monitoring +2025-01-07 18:23:10,487 DEBUG HandlerThread:2211704 [system_info.py:probe():152] Probing system +2025-01-07 18:23:10,488 DEBUG HandlerThread:2211704 [system_info.py:_probe_git():137] Probing git +2025-01-07 18:23:10,493 DEBUG HandlerThread:2211704 [system_info.py:_probe_git():145] Probing git done +2025-01-07 18:23:10,493 DEBUG HandlerThread:2211704 [system_info.py:probe():200] Probing system done +2025-01-07 18:23:10,493 DEBUG HandlerThread:2211704 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-07T23:23:10.487545', 'startedAt': '2025-01-07T23:23:10.063101', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.6589999999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.277, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 838.028434753418}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 18:23:10,493 INFO HandlerThread:2211704 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 18:23:10,493 INFO HandlerThread:2211704 [system_monitor.py:probe():227] Publishing system info +2025-01-07 18:23:10,493 DEBUG HandlerThread:2211704 [system_info.py:_save_code():44] Saving code +2025-01-07 18:23:10,495 DEBUG HandlerThread:2211704 [system_info.py:_save_code():67] Saving code done +2025-01-07 18:23:10,495 DEBUG HandlerThread:2211704 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 18:23:10,556 DEBUG HandlerThread:2211704 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 18:23:10,557 INFO HandlerThread:2211704 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 18:23:10,560 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:23:10,560 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 18:23:10,560 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 18:23:10,561 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 18:23:10,807 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/tv42cjfj-wandb-metadata.json +2025-01-07 18:23:10,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 18:23:10,813 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: python_packages +2025-01-07 18:23:10,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:23:10,813 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:23:10,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:10,816 INFO HandlerThread:2211704 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/tb" + save: true +} +control { + relay_id: "d6d220aa51274965a20635b41aeddf86" +} +_info { + stream_id: "no798ka2" +} + +2025-01-07 18:23:10,817 DEBUG HandlerThread:2211704 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 18:23:10,818 INFO wandb-upload_2:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/htub1ma4-diff.patch +2025-01-07 18:23:10,842 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/n93w829l-code/FISH/eval_robot.py +2025-01-07 18:23:10,916 DEBUG SenderThread:2211704 [sender.py:send():379] send: telemetry +2025-01-07 18:23:10,916 DEBUG SenderThread:2211704 [sender.py:send():379] send: tbrecord +2025-01-07 18:23:10,917 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:23:10,917 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 with policy live +2025-01-07 18:23:11,144 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/2j7sy6aw-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:23:11,372 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-metadata.json +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/code/FISH/eval_robot.py +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/requirements.txt +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/diff.patch +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/code/FISH +2025-01-07 18:23:11,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/code +2025-01-07 18:23:11,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:12,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:13,372 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:13,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:14,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:15,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:15,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:16,495 DEBUG SenderThread:2211704 [sender.py:send():379] send: config +2025-01-07 18:23:16,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:17,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:18,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:19,373 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:19,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:20,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:20,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:21,374 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:21,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:22,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:23,375 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:23,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:24,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:25,375 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:25,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:23:25,813 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:23:25,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:25,925 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:26,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:27,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:28,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:29,376 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:29,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:30,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:31,377 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:31,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:31,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:32,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:33,378 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:23:33,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:34,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:35,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:36,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:36,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:37,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:38,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:39,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:40,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:23:40,814 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:23:40,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:41,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:41,948 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:42,380 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml +2025-01-07 18:23:42,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:43,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:44,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:45,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:46,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:47,814 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:47,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:48,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:49,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:50,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:51,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:52,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:52,818 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:53,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:54,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:55,813 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:23:55,814 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:23:55,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:56,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:57,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:57,936 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:23:58,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:23:59,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:00,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:01,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:02,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:03,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:03,821 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:04,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:05,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:06,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:07,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:07,961 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:24:07,962 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:24:07,962 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 with policy now +2025-01-07 18:24:07,962 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:24:07,962 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:24:07,963 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:24:08,233 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/udm3crnc-media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media +2025-01-07 18:24:08,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:24:08,687 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/9p7ujrt5-media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 +2025-01-07 18:24:08,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:08,964 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:09,387 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:09,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:10,477 DEBUG SystemMonitor:2211704 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 18:24:10,478 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:24:10,819 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:24:10,819 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:24:10,861 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:11,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:12,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:13,389 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:13,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:14,354 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:14,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:15,389 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:15,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:16,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:17,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:18,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:19,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:19,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:20,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:21,391 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:24:21,657 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/pie1o1m1-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:24:21,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:22,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:23,392 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:23,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:24,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:24,950 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:25,815 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:24:25,815 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:24:25,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:26,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:27,393 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:24:27,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:28,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:29,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:30,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:30,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:31,825 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:32,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:33,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:34,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:35,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:35,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:36,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:37,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:38,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:39,827 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:40,478 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:24:40,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:24:40,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:24:40,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:40,943 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:41,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:42,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:43,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:44,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:45,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:46,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:46,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:47,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:48,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:49,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:50,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:51,817 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:51,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:52,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:53,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:54,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:55,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:24:55,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:24:55,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:56,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:56,896 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:24:57,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:58,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:24:59,829 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:00,657 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:25:00,719 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:25:00,719 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 with policy now +2025-01-07 18:25:00,720 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:25:00,720 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:25:00,721 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:25:00,834 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:00,993 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/0hpy56cy-media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 +2025-01-07 18:25:01,405 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:25:01,405 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:01,405 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 +2025-01-07 18:25:01,406 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:25:01,714 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/k1oiv7il-media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 +2025-01-07 18:25:01,832 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:02,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:02,838 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:03,406 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:03,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:04,836 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:05,406 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:05,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:06,837 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:07,835 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:08,217 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:08,836 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:09,408 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:09,836 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:10,479 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:25:10,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:25:10,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:25:10,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:11,837 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:12,837 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:13,839 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:13,874 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:14,838 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:15,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:16,838 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:17,839 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:18,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:19,357 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:19,411 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:25:19,839 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:20,242 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:25:20,243 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:25:20,244 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:25:20,411 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:25:20,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:21,412 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:21,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:22,840 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:23,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:24,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:25,341 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:25,413 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:25,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:25:25,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:25:25,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:26,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:27,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:28,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:29,841 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:30,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:30,844 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:31,844 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:32,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:33,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:34,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:35,824 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:35,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:36,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:37,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:38,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:39,845 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:40,479 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:25:40,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:25:40,816 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:25:40,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:40,967 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:41,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:42,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:43,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:44,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:45,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:45,997 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:46,846 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:47,420 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:47,847 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:48,847 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:49,421 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:49,847 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:50,847 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:51,826 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:51,848 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:52,849 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:53,848 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:54,848 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:55,816 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:25:55,817 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:25:55,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:56,848 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:56,946 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:25:57,849 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:58,860 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:25:58,981 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:25:58,981 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:25:58,981 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 with policy now +2025-01-07 18:25:58,982 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:25:58,982 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:25:58,999 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:25:59,256 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/o4dox3pn-media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 +2025-01-07 18:25:59,424 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:25:59,425 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:25:59,425 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 +2025-01-07 18:25:59,425 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:25:59,708 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/1v94weyc-media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 +2025-01-07 18:25:59,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:00,853 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:01,425 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:01,857 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:02,705 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:02,861 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:03,860 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:04,866 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:05,427 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:05,861 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:06,863 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:07,427 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:07,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:07,870 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:08,867 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:09,873 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:10,480 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:26:10,822 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:26:10,823 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:26:10,874 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:11,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:12,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:12,973 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:13,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:14,088 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:26:14,088 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:26:14,089 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:26:14,430 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:26:14,430 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:26:14,720 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/vrrxb54c-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:26:14,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:15,430 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:15,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:16,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:17,431 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:17,874 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:17,976 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:18,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:19,431 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:19,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:20,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:21,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:22,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:23,830 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:23,875 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:24,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:25,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:26:25,820 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:26:25,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:26,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:27,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:28,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:28,913 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:29,876 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:30,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:31,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:32,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:33,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:34,689 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:34,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:35,437 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:35,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:36,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:37,438 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:37,877 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:38,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:39,831 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:39,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:40,480 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:26:40,820 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:26:40,820 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:26:40,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:41,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:42,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:43,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:44,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:44,890 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:45,878 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:46,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:47,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:48,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:49,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:50,831 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:50,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:51,879 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:52,885 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:53,443 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:53,882 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:54,016 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:26:54,016 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 with policy now +2025-01-07 18:26:54,020 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:26:54,020 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:26:54,020 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:26:54,026 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:26:54,279 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/xjbh0wj3-media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 +2025-01-07 18:26:54,443 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:26:54,444 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 +2025-01-07 18:26:54,444 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:26:54,745 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/nob6d7t0-media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 +2025-01-07 18:26:54,884 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:55,444 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:55,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:26:55,833 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:26:55,890 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:55,910 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:26:56,444 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:56,885 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:57,885 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:58,892 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:26:59,445 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:26:59,891 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:00,882 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:01,446 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:01,832 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:01,895 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:02,889 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:03,892 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:04,889 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:05,895 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:06,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:06,896 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:07,897 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:08,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:09,155 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:27:09,155 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:27:09,156 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:27:09,449 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:27:09,449 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:27:09,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:10,481 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:27:10,835 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:27:10,835 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:27:10,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:11,449 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:11,903 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:11,951 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:12,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:13,897 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:14,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:15,451 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:15,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:16,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:17,259 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:17,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:18,898 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:19,452 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:19,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:20,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:21,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:22,832 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:22,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:23,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:24,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:25,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:27:25,823 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:27:25,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:26,899 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:27,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:27,931 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:28,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:29,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:30,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:31,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:32,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:33,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:33,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:34,900 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:35,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:36,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:37,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:38,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:39,833 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:39,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:40,481 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:27:40,823 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:27:40,823 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:27:40,901 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:41,903 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:42,902 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:43,902 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:44,902 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:44,959 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:45,902 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:46,903 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:47,903 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:48,906 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:48,951 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:27:48,951 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 with policy now +2025-01-07 18:27:48,993 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:27:48,994 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:27:48,994 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:27:48,995 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:27:49,235 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/ytpxu9di-media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 +2025-01-07 18:27:49,462 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:27:49,463 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:49,463 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 +2025-01-07 18:27:49,463 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval +2025-01-07 18:27:49,743 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/8uwvw4rl-media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 +2025-01-07 18:27:49,911 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:49,996 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:50,915 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:51,463 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:51,906 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:52,930 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:53,464 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:53,917 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:54,914 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:55,464 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:55,499 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:27:55,828 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 18:27:55,829 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: stop_status +2025-01-07 18:27:55,923 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:56,921 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:57,465 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:27:57,927 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:58,925 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:27:59,216 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:27:59,216 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:27:59,217 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:27:59,466 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:27:59,466 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:27:59,518 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:27:59,518 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:27:59,674 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:27:59,727 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/xfeu384j-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:27:59,921 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:00,007 INFO wandb-upload_0:2211704 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpmqqm3ukd +2025-01-07 18:28:00,076 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 18:28:00,087 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 18:28:00,402 INFO SenderThread:2211704 [sender.py:send_request_log_artifact():1525] logged artifact run-no798ka2-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjExMzE3Ng==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc1NTkyOA==', 'latestArtifact': None}} +2025-01-07 18:28:00,402 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:28:00,402 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json with policy now +2025-01-07 18:28:00,402 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:28:00,402 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,403 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,403 DEBUG SenderThread:2211704 [sender.py:send():379] send: telemetry +2025-01-07 18:28:00,403 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,407 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,407 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,408 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,408 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,408 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,408 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,409 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,409 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,409 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,409 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,410 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,410 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,410 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,410 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,410 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,411 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,411 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,411 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,411 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,412 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:00,412 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:00,412 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 18:28:00,466 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:00,466 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval +2025-01-07 18:28:00,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table +2025-01-07 18:28:00,585 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/h7qe27r4-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 18:28:00,676 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/6hy7j2ag-media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json +2025-01-07 18:28:00,774 INFO wandb-upload_2:2211704 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp00tumeh3 +2025-01-07 18:28:00,846 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/89tlbp6u-media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 18:28:00,922 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:01,166 INFO SenderThread:2211704 [sender.py:send_request_log_artifact():1525] logged artifact run-no798ka2-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjExMzIwMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTc1NTkzMA==', 'latestArtifact': None}} +2025-01-07 18:28:01,170 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:01,275 DEBUG SenderThread:2211704 [sender.py:send():379] send: files +2025-01-07 18:28:01,275 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json with policy now +2025-01-07 18:28:01,275 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:28:01,275 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:01,275 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:01,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:01,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml +2025-01-07 18:28:01,467 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:28:01,531 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/175n4i24-media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json +2025-01-07 18:28:01,922 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:02,505 DEBUG SenderThread:2211704 [sender.py:send():379] send: exit +2025-01-07 18:28:02,505 INFO SenderThread:2211704 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 18:28:02,506 INFO SenderThread:2211704 [sender.py:send_exit():588] handling runtime: 292 +2025-01-07 18:28:02,506 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:02,506 INFO SenderThread:2211704 [sender.py:send_exit():594] send defer +2025-01-07 18:28:02,506 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,506 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 18:28:02,506 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:02,506 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 18:28:02,507 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 1 +2025-01-07 18:28:02,507 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,507 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 18:28:02,507 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:02,507 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 18:28:02,507 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 2 +2025-01-07 18:28:02,507 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,507 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 18:28:02,507 INFO HandlerThread:2211704 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 18:28:02,507 DEBUG SystemMonitor:2211704 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 18:28:02,507 DEBUG SystemMonitor:2211704 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 18:28:02,508 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 18:28:02,508 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 18:28:02,559 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 18:28:02,559 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 18:28:02,559 INFO HandlerThread:2211704 [interfaces.py:finish():200] Joined network monitor +2025-01-07 18:28:02,560 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:02,560 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 18:28:02,560 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 3 +2025-01-07 18:28:02,560 DEBUG SenderThread:2211704 [sender.py:send():379] send: stats +2025-01-07 18:28:02,560 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,560 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 18:28:02,561 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:02,561 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 18:28:02,561 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 4 +2025-01-07 18:28:02,561 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:02,561 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 18:28:03,468 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:03,468 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:28:09,222 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:09,222 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:09,222 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 18:28:09,224 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:09,224 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 18:28:09,224 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 5 +2025-01-07 18:28:09,224 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:09,224 DEBUG SenderThread:2211704 [sender.py:send():379] send: history +2025-01-07 18:28:09,224 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:09,224 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: summary_record +2025-01-07 18:28:09,224 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 18:28:09,225 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:09,225 DEBUG SenderThread:2211704 [sender.py:send():379] send: summary +2025-01-07 18:28:09,226 INFO SenderThread:2211704 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 18:28:09,226 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:09,226 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 18:28:09,226 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 6 +2025-01-07 18:28:09,226 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:09,226 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 18:28:09,226 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:09,226 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 18:28:09,227 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 7 +2025-01-07 18:28:09,227 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 18:28:09,227 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:09,227 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 18:28:09,227 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:09,227 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 18:28:09,470 INFO Thread-12 :2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:09,507 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:12,037 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 8 +2025-01-07 18:28:12,037 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:12,037 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,037 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 18:28:12,038 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,038 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 18:28:12,038 INFO SenderThread:2211704 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 18:28:12,040 INFO SenderThread:2211704 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 18:28:12,042 INFO SenderThread:2211704 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 18:28:12,053 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 9 +2025-01-07 18:28:12,054 DEBUG SenderThread:2211704 [sender.py:send():379] send: artifact +2025-01-07 18:28:12,056 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,056 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 18:28:12,238 INFO SenderThread:2211704 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNTkyMzk5OA==', 'versionIndex': 436}}} +2025-01-07 18:28:12,238 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,238 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 18:28:12,238 INFO SenderThread:2211704 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 18:28:12,471 INFO SenderThread:2211704 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:28:12,471 INFO SenderThread:2211704 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files +2025-01-07 18:28:12,471 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml config.yaml +2025-01-07 18:28:12,472 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json wandb-summary.json +2025-01-07 18:28:12,472 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/requirements.txt requirements.txt +2025-01-07 18:28:12,472 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-metadata.json wandb-metadata.json +2025-01-07 18:28:12,472 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:28:12,476 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log output.log +2025-01-07 18:28:12,476 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/diff.patch diff.patch +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 media/videos/eval/0_eval_0_c3c69d41167ae5a9086a.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 media/videos/eval/1_eval_1_ddb2bcf1bbe47f8bb323.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 media/videos/eval/3_eval_3_3a5d4eef35ecdfe13399.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 media/videos/eval/2_eval_2_05d28ce3514e2b70c559.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 media/videos/eval/4_eval_4_e8bdad0cca02a9e00142.mp4 +2025-01-07 18:28:12,477 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json media/table/eval/success_rate_ci_5_d8821c50291f0fbf979f.table.json +2025-01-07 18:28:12,478 INFO SenderThread:2211704 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json media/table/eval/total_success_rate_ci_6_ac8537acf8bec14e3652.table.json +2025-01-07 18:28:12,478 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 10 +2025-01-07 18:28:12,480 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,481 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 18:28:12,481 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,481 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 18:28:12,481 INFO SenderThread:2211704 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:28:12,508 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:12,508 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:12,655 INFO wandb-upload_0:2211704 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/requirements.txt +2025-01-07 18:28:12,661 INFO wandb-upload_1:2211704 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/config.yaml +2025-01-07 18:28:12,672 INFO wandb-upload_2:2211704 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/wandb-summary.json +2025-01-07 18:28:12,688 INFO wandb-upload_4:2211704 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/files/output.log +2025-01-07 18:28:12,756 INFO wandb-upload_3:2211704 [upload_job.py:push():130] Uploaded file /tmp/tmpkj6e75kawandb/9acx95ix-events.out.tfevents.1736292190.leonmkim-ROG-Strix-G15CS-G15CS.2211530.0 +2025-01-07 18:28:12,957 INFO Thread-11 (_thread_body):2211704 [sender.py:transition_state():614] send defer: 11 +2025-01-07 18:28:12,957 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,957 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 18:28:12,957 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,957 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 18:28:12,957 INFO SenderThread:2211704 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 18:28:12,959 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 12 +2025-01-07 18:28:12,959 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:12,959 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 18:28:12,959 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:12,959 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 18:28:12,959 INFO SenderThread:2211704 [file_stream.py:finish():601] file stream finish called +2025-01-07 18:28:13,160 INFO SenderThread:2211704 [file_stream.py:finish():605] file stream finish is done +2025-01-07 18:28:13,160 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 13 +2025-01-07 18:28:13,160 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:13,160 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 18:28:13,160 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:13,160 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 18:28:13,160 INFO SenderThread:2211704 [sender.py:transition_state():614] send defer: 14 +2025-01-07 18:28:13,160 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: defer +2025-01-07 18:28:13,160 INFO HandlerThread:2211704 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 18:28:13,161 DEBUG SenderThread:2211704 [sender.py:send():379] send: final +2025-01-07 18:28:13,161 DEBUG SenderThread:2211704 [sender.py:send():379] send: footer +2025-01-07 18:28:13,161 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: defer +2025-01-07 18:28:13,161 INFO SenderThread:2211704 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 18:28:13,161 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:13,161 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 18:28:13,161 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 18:28:13,162 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:13,162 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: server_info +2025-01-07 18:28:13,164 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 18:28:13,164 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 18:28:13,165 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 18:28:13,210 DEBUG SenderThread:2211704 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 18:28:13,210 INFO MainThread:2211704 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 18:28:13,211 INFO MainThread:2211704 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 18:28:13,211 INFO MainThread:2211704 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 18:28:13,211 DEBUG HandlerThread:2211704 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 18:28:13,211 INFO HandlerThread:2211704 [handler.py:finish():882] shutting down handler +2025-01-07 18:28:14,162 INFO WriterThread:2211704 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/run-no798ka2.wandb +2025-01-07 18:28:14,210 INFO SenderThread:2211704 [sender.py:finish():1615] shutting down sender +2025-01-07 18:28:14,210 INFO SenderThread:2211704 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 18:28:14,210 INFO SenderThread:2211704 [file_pusher.py:join():175] waiting for file pusher diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug.log b/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..825c2fc5306ed34c493b4564d261c06a260bbddb --- /dev/null +++ b/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Configure stats pid to 2211530 +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/settings +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 18:23:10,066 INFO MainThread:2211530 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug.log +2025-01-07 18:23:10,067 INFO MainThread:2211530 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/wandb/run-20250107_182310-no798ka2/logs/debug-internal.log +2025-01-07 18:23:10,067 INFO MainThread:2211530 [wandb_init.py:init():569] calling init triggers +2025-01-07 18:23:10,067 INFO MainThread:2211530 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2'} +2025-01-07 18:23:10,068 INFO MainThread:2211530 [wandb_init.py:init():619] starting backend +2025-01-07 18:23:10,068 INFO MainThread:2211530 [wandb_init.py:init():623] setting up manager +2025-01-07 18:23:10,071 INFO MainThread:2211530 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 18:23:10,073 INFO MainThread:2211530 [wandb_init.py:init():631] backend started and connected +2025-01-07 18:23:10,090 INFO MainThread:2211530 [wandb_init.py:init():720] updated telemetry +2025-01-07 18:23:10,096 INFO MainThread:2211530 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 18:23:10,390 INFO MainThread:2211530 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 18:23:10,460 INFO MainThread:2211530 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 18:23:10,460 INFO MainThread:2211530 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 18:23:10,813 INFO MainThread:2211530 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 18:23:10,813 INFO MainThread:2211530 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 18:23:10,813 INFO MainThread:2211530 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 18:23:10,813 INFO MainThread:2211530 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 18:23:10,816 INFO MainThread:2211530 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 18:23:10,816 INFO MainThread:2211530 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/no798ka2/tb, True +2025-01-07 18:23:16,494 INFO MainThread:2211530 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'lib', 'left_book_slot': 'twodim'} +2025-01-07 18:28:14,212 WARNING MsgRouterThr:2211530 [router.py:message_loop():77] message_loop has been closed diff --git a/no798ka2/wandb/run-20250107_182310-no798ka2/run-no798ka2.wandb b/no798ka2/wandb/run-20250107_182310-no798ka2/run-no798ka2.wandb new file mode 100644 index 0000000000000000000000000000000000000000..bd027f697d58e5251eaaeb971910bfcf2c2e82c1 Binary files /dev/null and b/no798ka2/wandb/run-20250107_182310-no798ka2/run-no798ka2.wandb differ diff --git a/q50r8fwa/.hydra/config.yaml b/q50r8fwa/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3 --- /dev/null +++ b/q50r8fwa/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1013_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/q50r8fwa/.hydra/hydra.yaml b/q50r8fwa/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86aaac43c15eb6f94cac5befae662ec473ee25c2 --- /dev/null +++ b/q50r8fwa/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/q50r8fwa/.hydra/overrides.yaml b/q50r8fwa/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/q50r8fwa/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/q50r8fwa/episode_rosbags/aligned_depth_to_color_K.npy b/q50r8fwa/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/q50r8fwa/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/q50r8fwa/episode_rosbags/cam_tf_world.npy b/q50r8fwa/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/q50r8fwa/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/q50r8fwa/episode_rosbags/color_K.npy b/q50r8fwa/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/q50r8fwa/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/q50r8fwa/episode_rosbags/depth_K.npy b/q50r8fwa/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/q50r8fwa/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/q50r8fwa/eval_robot.log b/q50r8fwa/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..6ba6b5133fb9f8b613199799f790371adf07538e --- /dev/null +++ b/q50r8fwa/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 20:16:10,009][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:16:10,012][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/q50r8fwa/eval_video/0_eval.mp4 b/q50r8fwa/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ccd36f1b45cbef62a3ade51b20ec5d0d19fb7cf4 --- /dev/null +++ b/q50r8fwa/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd301aef3dc772a0c0e9a6fa4fc3085d943533957948129f00f5979b9ffd1681 +size 723458 diff --git a/q50r8fwa/eval_video/1_eval.mp4 b/q50r8fwa/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..958fab3226034ed518ab100954cba86dbb6c354f --- /dev/null +++ b/q50r8fwa/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77e85503de0c92b9ff4c6bb8563e3659b1e0fad00ababa5df43f9804ab53de51 +size 596530 diff --git a/q50r8fwa/eval_video/2_eval.mp4 b/q50r8fwa/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3724de05934b4f82122d59ef5a32870020caa237 --- /dev/null +++ b/q50r8fwa/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2011ea908dfd24dcf4b7465954307a84c0f785a14213aa0faaf416172dc56302 +size 1471675 diff --git a/q50r8fwa/eval_video/3_eval.mp4 b/q50r8fwa/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e90e71b4104ce422924490d5cfec08132ea8d213 --- /dev/null +++ b/q50r8fwa/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c14c10349ec35d2de07af10177ac61ad953ef0509b67ec8b4202ff0aec7d5ede +size 1522101 diff --git a/q50r8fwa/eval_video/4_eval.mp4 b/q50r8fwa/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..441159fd9c9a21b5d718ca33ecf468ce2680c803 --- /dev/null +++ b/q50r8fwa/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84c6290cf5d078bb7ea707febc1b18a7112acacf61f3aa0a667439b45013bd5f +size 1485602 diff --git a/q50r8fwa/tb/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 b/q50r8fwa/tb/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 new file mode 100644 index 0000000000000000000000000000000000000000..b97dac0c6512c53cd546bdf8d60049d2fa11c29a --- /dev/null +++ b/q50r8fwa/tb/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2017bf57ad83c9498f846759a2fae2eca6cfaca73ea47bf0c7d19336f2148be0 +size 6499 diff --git a/q50r8fwa/wandb/debug-internal.log b/q50r8fwa/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7cc3688a316349a64a920fb41f2b0ed9b4ad71ae --- /dev/null +++ b/q50r8fwa/wandb/debug-internal.log @@ -0,0 +1,773 @@ +2025-01-07 20:16:15,738 INFO StreamThr :2222977 [internal.py:wandb_internal():85] W&B internal server running at pid: 2222977, started at: 2025-01-07 20:16:15.738368 +2025-01-07 20:16:15,740 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:16:15,754 INFO WriterThread:2222977 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/run-q50r8fwa.wandb +2025-01-07 20:16:15,754 DEBUG SenderThread:2222977 [sender.py:send():379] send: header +2025-01-07 20:16:15,761 DEBUG SenderThread:2222977 [sender.py:send():379] send: run +2025-01-07 20:16:16,079 INFO SenderThread:2222977 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files +2025-01-07 20:16:16,079 INFO SenderThread:2222977 [sender.py:_start_run_threads():1188] run started: q50r8fwa with start time 1736298975.739403 +2025-01-07 20:16:16,095 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:16:16,096 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:16:16,140 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:16:16,145 DEBUG HandlerThread:2222977 [system_info.py:__init__():26] System info init +2025-01-07 20:16:16,145 DEBUG HandlerThread:2222977 [system_info.py:__init__():41] System info init done +2025-01-07 20:16:16,145 INFO HandlerThread:2222977 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:16:16,145 INFO SystemMonitor:2222977 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:16:16,146 INFO HandlerThread:2222977 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:16:16,146 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:16:16,147 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:16:16,147 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:16:16,148 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:16:16,149 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:16:16,159 DEBUG HandlerThread:2222977 [system_info.py:probe():152] Probing system +2025-01-07 20:16:16,160 DEBUG HandlerThread:2222977 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:16:16,165 DEBUG HandlerThread:2222977 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:16:16,165 DEBUG HandlerThread:2222977 [system_info.py:probe():200] Probing system done +2025-01-07 20:16:16,165 DEBUG HandlerThread:2222977 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:16:16.159353', 'startedAt': '2025-01-08T01:16:15.732198', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.2597499999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.364, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.936, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 563.400520324707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:16:16,165 INFO HandlerThread:2222977 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:16:16,165 INFO HandlerThread:2222977 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:16:16,165 DEBUG HandlerThread:2222977 [system_info.py:_save_code():44] Saving code +2025-01-07 20:16:16,167 DEBUG HandlerThread:2222977 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:16:16,167 DEBUG HandlerThread:2222977 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:16:16,230 DEBUG HandlerThread:2222977 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:16:16,230 INFO HandlerThread:2222977 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:16:16,233 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:16:16,233 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:16:16,233 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:16:16,234 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:16:16,485 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:16:16,485 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:16:16,487 DEBUG SenderThread:2222977 [sender.py:send():379] send: telemetry +2025-01-07 20:16:16,500 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/ph1tuwzo-diff.patch +2025-01-07 20:16:16,501 INFO wandb-upload_0:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/wf4ttdkm-wandb-metadata.json +2025-01-07 20:16:16,517 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/5cyb1ogy-code/FISH/eval_robot.py +2025-01-07 20:16:16,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:16:16,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:16,529 INFO HandlerThread:2222977 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/tb" + save: true +} +control { + relay_id: "b561f5dd91de41acb6506c805f35aec4" +} +_info { + stream_id: "q50r8fwa" +} + +2025-01-07 20:16:16,529 DEBUG HandlerThread:2222977 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:16:16,531 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:16:16,623 DEBUG SenderThread:2222977 [sender.py:send():379] send: tbrecord +2025-01-07 20:16:16,623 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:16:16,623 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 with policy live +2025-01-07 20:16:16,804 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/xsx4ygmv-events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:16:17,081 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/diff.patch +2025-01-07 20:16:17,081 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:16:17,081 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:17,081 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/requirements.txt +2025-01-07 20:16:17,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-metadata.json +2025-01-07 20:16:17,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code/FISH/eval_robot.py +2025-01-07 20:16:17,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code/FISH +2025-01-07 20:16:17,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code +2025-01-07 20:16:17,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:18,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:19,080 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:19,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:20,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:21,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:21,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:22,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:23,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:23,929 DEBUG SenderThread:2222977 [sender.py:send():379] send: config +2025-01-07 20:16:24,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:25,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:25,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:26,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:26,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:27,083 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:27,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:28,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:29,083 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:29,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:30,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:31,084 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:31,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:16:31,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:16:31,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:31,605 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:32,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:33,085 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:33,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:34,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:35,085 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:35,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:36,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:36,851 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:37,086 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:37,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:38,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:39,086 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:39,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:40,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:41,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:42,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:42,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:43,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:44,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:45,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:46,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:16:46,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:16:46,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:47,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:47,615 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:48,089 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml +2025-01-07 20:16:48,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:49,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:50,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:51,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:52,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:52,747 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:53,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:53,713 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:16:53,714 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:16:53,714 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 with policy now +2025-01-07 20:16:53,714 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:16:53,714 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:16:53,716 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:16:53,966 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/2wxp72mb-media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 +2025-01-07 20:16:54,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 +2025-01-07 20:16:54,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:16:54,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media +2025-01-07 20:16:54,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos +2025-01-07 20:16:54,092 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:16:54,358 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/bbxcq12e-media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 +2025-01-07 20:16:54,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:55,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:55,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:56,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:57,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:58,349 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:58,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:59,093 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:59,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:00,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:01,093 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:01,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:17:01,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:17:01,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:02,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:03,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:03,633 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:04,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:05,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:06,095 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:17:06,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:07,095 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:07,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:08,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:09,096 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:09,136 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:09,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:10,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:11,097 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:11,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:12,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:13,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:14,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:14,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:15,497 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:16,149 DEBUG SystemMonitor:2222977 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:17:16,150 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:17:16,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:17:16,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:17:16,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:17,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:18,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:19,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:19,617 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:20,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:21,537 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:22,498 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:22,833 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:17:22,834 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:17:22,834 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 with policy now +2025-01-07 20:17:22,841 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:17:22,841 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:17:22,841 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:17:23,078 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/a0gwq0is-media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 +2025-01-07 20:17:23,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:17:23,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:23,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 +2025-01-07 20:17:23,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:17:23,354 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/cwj5csrf-media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 +2025-01-07 20:17:23,498 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:24,501 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:24,846 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:25,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:25,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:26,501 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:27,102 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:27,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:28,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:29,501 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:30,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:30,773 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:31,103 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:31,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:17:31,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:17:31,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:32,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:33,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:34,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:35,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:36,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:36,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:37,503 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:38,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:39,503 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:40,504 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:41,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:41,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:42,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:43,503 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:44,503 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:45,504 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:46,150 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:17:46,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:17:46,490 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:17:46,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:46,556 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:47,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:48,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:48,916 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:17:48,916 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:17:48,916 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:17:49,111 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:17:49,111 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:17:49,355 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/7uom1kkc-events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:17:49,525 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:50,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:51,112 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:51,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:52,197 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:52,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:53,113 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:53,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:54,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:55,114 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:55,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:56,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:57,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:57,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:58,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:59,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:00,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:01,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:18:01,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:18:01,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:02,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:02,554 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:03,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:04,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:05,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:06,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:07,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:08,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:08,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:09,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:10,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:11,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:12,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:13,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:14,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:14,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:15,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:16,151 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:18:16,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:18:16,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:18:16,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:17,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:18,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:19,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:19,582 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:20,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:21,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:22,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:23,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:24,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:25,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:25,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:26,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:27,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:28,512 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:29,124 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:29,521 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:29,543 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:18:29,543 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 with policy now +2025-01-07 20:18:29,585 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:18:29,588 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:18:29,588 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:18:29,596 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:18:29,802 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/52jaumdd-media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 +2025-01-07 20:18:30,125 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:18:30,125 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 +2025-01-07 20:18:30,125 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:18:30,418 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/o7mnof7h-media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 +2025-01-07 20:18:30,516 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:30,617 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:31,125 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:31,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:18:31,492 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:18:31,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:32,126 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:32,532 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:33,527 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:34,525 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:35,127 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:35,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:36,172 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:36,524 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:37,127 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:37,531 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:38,531 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:39,528 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:39,756 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:18:39,756 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:18:39,757 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:18:40,128 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:18:40,129 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:18:40,532 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:41,129 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:41,441 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:41,532 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:42,532 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:43,129 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:43,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:44,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:45,130 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:45,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:46,151 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:18:46,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:18:46,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:18:46,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:46,602 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:47,131 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:47,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:48,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:49,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:50,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:51,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:52,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:52,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:53,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:54,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:55,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:56,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:57,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:58,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:58,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:59,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:00,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:01,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:19:01,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:19:01,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:02,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:03,136 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:03,537 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:04,045 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:04,537 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:05,537 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:06,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:07,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:08,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:09,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:09,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:10,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:11,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:12,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:13,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:14,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:14,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:15,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:16,152 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:19:16,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:19:16,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:19:16,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:17,540 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:18,540 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:19,540 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:19,644 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:20,557 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:21,549 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:22,011 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:19:22,011 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 with policy now +2025-01-07 20:19:22,015 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:19:22,015 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:19:22,015 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:19:22,016 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:19:22,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:19:22,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 +2025-01-07 20:19:22,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:19:22,275 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/ocp1hzz3-media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 +2025-01-07 20:19:22,542 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:22,587 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/9dgm77nd-media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 +2025-01-07 20:19:23,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:23,546 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:24,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:24,558 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:25,058 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:25,554 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:26,552 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:27,145 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:27,556 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:28,553 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:29,552 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:30,558 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:30,613 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:31,147 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:31,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:19:31,493 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:19:31,558 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:32,559 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:33,557 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:34,557 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:35,556 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:36,514 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:36,557 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:37,558 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:38,559 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:39,564 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:40,565 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:41,566 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:42,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:42,576 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:43,151 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:19:43,393 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/awgb8vwr-events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:19:43,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:43,922 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:19:43,922 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:19:43,923 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:19:44,151 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:19:44,578 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:45,152 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:45,578 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:46,152 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:19:46,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:19:46,490 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:19:46,575 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:47,567 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:47,754 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:48,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:49,154 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:49,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:50,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:51,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:52,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:53,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:53,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:54,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:55,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:56,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:57,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:58,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:59,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:59,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:00,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:01,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:20:01,489 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:20:01,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:02,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:03,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:04,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:04,611 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:05,571 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:06,571 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:07,571 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:08,571 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:09,572 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:10,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:10,572 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:11,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:12,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:13,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:14,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:15,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:16,152 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:20:16,153 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:16,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:20:16,489 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:20:16,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:17,163 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:17,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:18,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:19,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:20,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:21,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:21,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:22,574 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:23,577 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:23,956 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:20:23,956 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 with policy now +2025-01-07 20:20:23,960 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:20:23,961 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:23,961 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:23,961 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:24,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:24,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 +2025-01-07 20:20:24,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:20:24,262 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/9n9nd5sa-media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 +2025-01-07 20:20:24,541 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/o9n3dcfm-media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 +2025-01-07 20:20:24,577 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:25,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:25,579 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:26,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:26,520 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:26,578 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:27,580 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:28,580 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:29,167 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:29,578 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:30,583 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:31,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:20:31,497 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:20:31,584 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:31,632 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:32,579 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:33,169 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:33,582 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:34,585 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:35,590 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:36,596 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:37,514 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:37,599 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:38,598 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:39,599 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:40,602 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:41,605 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:42,007 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:42,007 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:42,008 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:42,172 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:20:42,172 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:42,605 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:42,641 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:20:42,641 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:20:42,824 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:20:43,138 INFO wandb-upload_1:2222977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpian69z5u +2025-01-07 20:20:43,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:43,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-07 20:20:43,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media +2025-01-07 20:20:43,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval +2025-01-07 20:20:43,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table +2025-01-07 20:20:43,308 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:20:43,316 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:20:43,574 INFO SenderThread:2222977 [sender.py:send_request_log_artifact():1525] logged artifact run-q50r8fwa-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM3NTUzMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyODk5Nw==', 'latestArtifact': None}} +2025-01-07 20:20:43,574 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:20:43,574 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json with policy now +2025-01-07 20:20:43,574 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:43,574 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:43,574 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,575 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,575 DEBUG SenderThread:2222977 [sender.py:send():379] send: telemetry +2025-01-07 20:20:43,575 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,579 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,579 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,580 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,580 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,580 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,580 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,581 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,581 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,581 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,581 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,582 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,582 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,582 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,582 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,583 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,583 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,583 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,583 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,583 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,584 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,584 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,584 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:20:43,601 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:43,761 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/cbz7vsyf-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-07 20:20:44,028 INFO wandb-upload_1:2222977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq1fkeuv9 +2025-01-07 20:20:44,093 INFO wandb-upload_0:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/2z7qu50a-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-07 20:20:44,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:20:44,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:44,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json +2025-01-07 20:20:44,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval +2025-01-07 20:20:44,579 INFO SenderThread:2222977 [sender.py:send_request_log_artifact():1525] logged artifact run-q50r8fwa-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM3NTU1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyOTAwMg==', 'latestArtifact': None}} +2025-01-07 20:20:44,579 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:20:44,579 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json with policy now +2025-01-07 20:20:44,581 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:44,581 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:44,582 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:44,601 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:44,755 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/3ptywkim-media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json +2025-01-07 20:20:44,833 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/x9x0b433-events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:20:45,001 INFO wandb-upload_0:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/qrd4opyq-media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json +2025-01-07 20:20:45,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:45,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:45,602 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:46,153 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:20:46,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:20:46,489 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:20:46,621 DEBUG SenderThread:2222977 [sender.py:send():379] send: exit +2025-01-07 20:20:46,621 INFO SenderThread:2222977 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:20:46,621 INFO SenderThread:2222977 [sender.py:send_exit():588] handling runtime: 270 +2025-01-07 20:20:46,622 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:46,622 INFO SenderThread:2222977 [sender.py:send_exit():594] send defer +2025-01-07 20:20:46,622 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,622 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:20:46,622 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:46,623 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:20:46,623 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:20:46,623 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,623 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:20:46,623 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:46,623 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:20:46,623 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:20:46,623 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,623 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:20:46,623 INFO HandlerThread:2222977 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:20:46,623 DEBUG SystemMonitor:2222977 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:20:46,624 DEBUG SystemMonitor:2222977 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:20:46,625 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:20:46,625 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:20:46,676 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:20:46,676 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:20:46,677 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:20:46,677 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:46,677 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:20:46,677 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:20:46,677 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:20:46,677 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,677 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:20:46,678 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:46,678 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:20:46,678 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:20:46,678 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,678 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:20:47,175 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:49,176 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:53,012 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:53,012 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:53,012 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:53,012 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:20:53,012 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:20:53,012 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:20:53,012 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:53,013 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:53,013 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:20:53,013 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:53,014 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:53,015 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:53,015 DEBUG SenderThread:2222977 [sender.py:send():379] send: summary +2025-01-07 20:20:53,016 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:53,016 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:53,016 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:20:53,016 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:20:53,016 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:53,016 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:20:53,016 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:53,016 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:20:53,020 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:53,177 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:53,301 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:20:53,301 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:53,301 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:20:53,301 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:53,301 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:20:53,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:54,177 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml +2025-01-07 20:20:55,559 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:20:55,559 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:55,559 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:55,560 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:20:55,560 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:55,560 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:20:55,560 INFO SenderThread:2222977 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:20:55,562 INFO SenderThread:2222977 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:20:55,564 INFO SenderThread:2222977 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:20:55,577 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:20:55,577 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:55,577 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:20:55,577 DEBUG SenderThread:2222977 [sender.py:send():379] send: artifact +2025-01-07 20:20:55,831 INFO SenderThread:2222977 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:20:55,831 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:55,831 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:20:55,831 INFO SenderThread:2222977 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:20:56,178 INFO SenderThread:2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:56,179 INFO SenderThread:2222977 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files +2025-01-07 20:20:56,179 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml config.yaml +2025-01-07 20:20:56,179 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json wandb-summary.json +2025-01-07 20:20:56,179 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/requirements.txt requirements.txt +2025-01-07 20:20:56,182 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:20:56,182 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:20:56,182 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log output.log +2025-01-07 20:20:56,182 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/diff.patch diff.patch +2025-01-07 20:20:56,185 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:20:56,186 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,187 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:20:56,187 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,187 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:20:56,187 INFO SenderThread:2222977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:20:56,364 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:56,367 INFO wandb-upload_0:2222977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/requirements.txt +2025-01-07 20:20:56,398 INFO wandb-upload_3:2222977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:56,416 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml +2025-01-07 20:20:56,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:56,540 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:56,617 INFO Thread-11 (_thread_body):2222977 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:20:56,617 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,617 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:20:56,617 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,617 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:20:56,617 INFO SenderThread:2222977 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:20:56,618 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:20:56,619 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,619 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:20:56,619 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,619 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:20:56,619 INFO SenderThread:2222977 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:20:56,806 INFO SenderThread:2222977 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:20:56,806 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:20:56,806 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,806 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:20:56,806 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,806 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:20:56,806 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:20:56,807 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,807 DEBUG SenderThread:2222977 [sender.py:send():379] send: final +2025-01-07 20:20:56,807 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:20:56,807 DEBUG SenderThread:2222977 [sender.py:send():379] send: footer +2025-01-07 20:20:56,807 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,807 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:20:56,807 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:56,808 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:56,808 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:20:56,808 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:56,808 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:20:56,808 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:20:56,810 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:20:56,811 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:56,855 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:56,855 INFO MainThread:2222977 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:20:56,856 INFO MainThread:2222977 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:20:56,856 INFO MainThread:2222977 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:20:56,856 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:20:56,856 INFO HandlerThread:2222977 [handler.py:finish():882] shutting down handler +2025-01-07 20:20:57,808 INFO WriterThread:2222977 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/run-q50r8fwa.wandb +2025-01-07 20:20:57,855 INFO SenderThread:2222977 [sender.py:finish():1615] shutting down sender +2025-01-07 20:20:57,856 INFO SenderThread:2222977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:20:57,856 INFO SenderThread:2222977 [file_pusher.py:join():175] waiting for file pusher diff --git a/q50r8fwa/wandb/debug.log b/q50r8fwa/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8aeacb153ed9d328cde59f493ce86984c3bd597c --- /dev/null +++ b/q50r8fwa/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Configure stats pid to 2222778 +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/settings +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug.log +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug-internal.log +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa'} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:init():619] starting backend +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:init():623] setting up manager +2025-01-07 20:16:15,737 INFO MainThread:2222778 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:16:15,739 INFO MainThread:2222778 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:16:15,753 INFO MainThread:2222778 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:16:15,759 INFO MainThread:2222778 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:16:16,095 INFO MainThread:2222778 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:16:16,136 INFO MainThread:2222778 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:16:16,136 INFO MainThread:2222778 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:16:16,486 INFO MainThread:2222778 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:16:16,486 INFO MainThread:2222778 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:16:16,486 INFO MainThread:2222778 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:16:16,486 INFO MainThread:2222778 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:16:16,489 INFO MainThread:2222778 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:16:16,490 INFO MainThread:2222778 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/tb, True +2025-01-07 20:16:23,929 INFO MainThread:2222778 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:20:57,858 WARNING MsgRouterThr:2222778 [router.py:message_loop():77] message_loop has been closed diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code/FISH/eval_robot.py b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a7819abc08ca5036477b10aaa8af838791f5fcf --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1013_1' +wandb_notes: + desc: null + value: 1013_1_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736298975 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/diff.patch b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..8adda0290b36965f42f4119e05fd13ea9507158f --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/diff.patch @@ -0,0 +1,15 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..1c0be7a 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -139,8 +139,8 @@ load_checkpoint: ${agent.load_checkpoint} + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' +-# wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json new file mode 100644 index 0000000000000000000000000000000000000000..91d1314b0c81ca7ced2bf379f5d1a3bb24013b63 --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.6, 0.20694816118379347, 0.9527384935904063, 0.20744311815390934, 0.9364891498407951, 0.2632499635693074, 0.8492476194398473, 0.24815547471314967, 0.8085234164574604]]} \ No newline at end of file diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json new file mode 100644 index 0000000000000000000000000000000000000000..72b115ed57ff2198609d7cfe7e667e9ff41c7b53 --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[37, 55, 0.6727272727272727, 0.526706188911819, 0.8014992176883421, 0.5372565612122409, 0.7817560433777918, 0.5558311007805268, 0.7704544662888351, 0.5955363474595208, 0.7525842189283649]]} \ No newline at end of file diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..ccd36f1b45cbef62a3ade51b20ec5d0d19fb7cf4 --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd301aef3dc772a0c0e9a6fa4fc3085d943533957948129f00f5979b9ffd1681 +size 723458 diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..958fab3226034ed518ab100954cba86dbb6c354f --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77e85503de0c92b9ff4c6bb8563e3659b1e0fad00ababa5df43f9804ab53de51 +size 596530 diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..3724de05934b4f82122d59ef5a32870020caa237 --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2011ea908dfd24dcf4b7465954307a84c0f785a14213aa0faaf416172dc56302 +size 1471675 diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e90e71b4104ce422924490d5cfec08132ea8d213 --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c14c10349ec35d2de07af10177ac61ad953ef0509b67ec8b4202ff0aec7d5ede +size 1522101 diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..441159fd9c9a21b5d718ca33ecf468ce2680c803 --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84c6290cf5d078bb7ea707febc1b18a7112acacf61f3aa0a667439b45013bd5f +size 1485602 diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..d6b186f98ffc165a105f16a17bb2cece331cef4b --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log @@ -0,0 +1,281 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736298983.929658]: resetting environment +[INFO] [1736298983.933765]: cleared current plan +[INFO] [1736298983.935014]: moving to home +[INFO] [1736298984.937534]: reached home +[INFO] [1736298984.939661]: reset action history +[INFO] [1736298987.651408]: environment reset +Starting episode 0 +[INFO] [1736298987.656133]: resetting environment +[INFO] [1736298987.659801]: cleared current plan +[INFO] [1736298987.660635]: moving to home +[INFO] [1736298988.662395]: reached home +[INFO] [1736298988.664419]: reset action history +[INFO] [1736298991.378354]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode![INFO] [1736298993.129459]: resetting environment +[INFO] [1736298993.137354]: cleared current plan +[INFO] [1736298993.138446]: moving to home +[INFO] [1736298994.140424]: reached home +[INFO] [1736298994.141592]: reset action history +[INFO] [1736298996.850577]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/episode_rosbags/episode_0_2025-01-07-20-16-36.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2222778.log +started roslaunch server http://158.130.50.37:40131/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2223084] +process[print_text-2]: started with pid [2223108] +process[rosbag_record-3]: started with pid [2223109] +started bagging! +For topic gripper_width: timestamp difference is 102822296 for nearest: 1736299012202702276 - target: 1736299012099879980 at idx 211 +preempting episode +[INFO] [1736299012.683410]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736299013.452050]: Stored episode 1. +[INFO] [1736299013.453165]: Saving video... +[INFO] [1736299013.671556]: Video saved! +Starting episode 1 +[INFO] [1736299013.684613]: resetting environment +[INFO] [1736299013.699172]: cleared current plan +[INFO] [1736299013.701738]: moving to home +[INFO] [1736299017.305258]: reached home +[INFO] [1736299017.305728]: reset action history +[INFO] [1736299020.018865]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299025.421530]: resetting environment +[INFO] [1736299025.425404]: cleared current plan +[INFO] [1736299025.425590]: moving to home +[INFO] [1736299026.427141]: reached home +[INFO] [1736299026.427608]: reset action history +[INFO] [1736299029.135618]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/episode_rosbags/episode_1_2025-01-07-20-17-09.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2222778.log +started roslaunch server http://158.130.50.37:41857/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2223233] +process[print_text-5]: started with pid [2223235] +process[rosbag_record-6]: started with pid [2223259] +started bagging! +preempting episode +[INFO] [1736299041.505999]: Storing episode... +[rosbag_record-6] killing on exit +[print_text-5] killing on exit +[pub_text-4] killing on exit +[INFO] [1736299042.232784]: Stored episode 2. +[INFO] [1736299042.234357]: Saving video... +[INFO] [1736299042.785694]: Video saved! +Starting episode 2 +[INFO] [1736299042.813271]: resetting environment +[INFO] [1736299042.838986]: cleared current plan +[INFO] [1736299042.840731]: moving to home +[INFO] [1736299045.949373]: reached home +[INFO] [1736299045.953156]: reset action history +[INFO] [1736299048.729392]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299068.470034]: resetting environment +[INFO] [1736299068.479132]: cleared current plan +[INFO] [1736299068.479775]: moving to home +[INFO] [1736299069.481575]: reached home +[INFO] [1736299069.483230]: reset action history +[INFO] [1736299072.191432]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/episode_rosbags/episode_2_2025-01-07-20-17-52.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2222778.log +started roslaunch server http://158.130.50.37:32877/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2223412] +process[print_text-8]: started with pid [2223413] +process[rosbag_record-9]: started with pid [2223414] +started bagging! +[INFO] [1736299108.082960]: Storing episode... +[rosbag_record-9] killing on exit +[print_text-8] killing on exit[pub_text-7] killing on exit +[INFO] [1736299108.858290]: Stored episode 3. +[INFO] [1736299108.864606]: Saving video... +[INFO] [1736299109.511084]: Video saved! +Starting episode 3 +[INFO] [1736299109.545820]: resetting environment +[INFO] [1736299109.612355]: cleared current plan +[INFO] [1736299109.617475]: moving to home +[WARN] [1736299109.433510]: Plan exhausted +[WARN] [1736299109.474958]: Plan exhausted +[WARN] [1736299109.515302]: Plan exhausted +[WARN] [1736299109.555208]: Plan exhausted +[WARN] [1736299109.594006]: Plan exhausted +[INFO] [1736299113.330567]: reached home +[INFO] [1736299113.336070]: reset action history +[INFO] [1736299116.171040]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299119.368617]: resetting environment +[INFO] [1736299119.392081]: cleared current plan +[INFO] [1736299119.393974]: moving to home +[INFO] [1736299120.398225]: reached home +[INFO] [1736299120.400600]: reset action history +[INFO] [1736299123.195267]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/episode_rosbags/episode_3_2025-01-07-20-18-43.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2222778.log +started roslaunch server http://158.130.50.37:35963/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2223560] +process[print_text-11]: started with pid [2223561] +process[rosbag_record-12]: started with pid [2223562] +started bagging! +For topic gripper_width: timestamp difference is 111934699 for nearest: 1736299142002655782 - target: 1736299141890721083 at idx 243 +[INFO] [1736299160.516191]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736299161.304549]: Stored episode 4. +[INFO] [1736299161.310727]: Saving video... +[INFO] [1736299161.961746]: Video saved! +Starting episode 4 +[INFO] [1736299162.017860]: resetting environment +[INFO] [1736299162.051047]: cleared current plan +[INFO] [1736299162.055442]: moving to home +[WARN] [1736299161.877686]: Plan exhausted +[WARN] [1736299161.917345]: Plan exhausted +[WARN] [1736299161.957476]: Plan exhausted +[WARN] [1736299161.994108]: Plan exhausted +[WARN] [1736299162.033318]: Plan exhausted +[INFO] [1736299165.769275]: reached home +[INFO] [1736299165.770632]: reset action history +[INFO] [1736299168.572810]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299183.004156]: resetting environment +[INFO] [1736299183.026577]: cleared current plan +[INFO] [1736299183.027457]: moving to home +[INFO] [1736299184.034795]: reached home +[INFO] [1736299184.038972]: reset action history +[INFO] [1736299186.830723]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/episode_rosbags/episode_4_2025-01-07-20-19-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2222778.log +started roslaunch server http://158.130.50.37:35275/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2223724] +process[print_text-14]: started with pid [2223725] +process[rosbag_record-15]: started with pid [2223726] +started bagging! +For topic gripper_width: timestamp difference is -175831331 for nearest: 1736299214902667032 - target: 1736299215078498363 at idx 249 +[INFO] [1736299222.458538]: Storing episode... +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[pub_text-13] killing on exit +[INFO] [1736299223.271475]: Stored episode 5. +[INFO] [1736299223.276926]: Saving video... +[INFO] [1736299223.901707]: Video saved! +[INFO] [1736299223.961080]: resetting environment +[INFO] [1736299224.016636]: cleared current plan +[INFO] [1736299224.019709]: moving to home +[WARN] [1736299223.773072]: Plan exhausted +[WARN] [1736299223.821033]: Plan exhausted +[WARN] [1736299223.853876]: Plan exhausted +[WARN] [1736299223.894941]: Plan exhausted +[WARN] [1736299223.932836]: Plan exhausted +[WARN] [1736299223.971845]: Plan exhausted +[WARN] [1736299224.011319]: Plan exhausted +[INFO] [1736299227.737762]: reached home +[INFO] [1736299227.747314]: reset action history +[INFO] [1736299230.598692]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/requirements.txt b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-metadata.json b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..bb4d8be526cff9536fd53d8774bd6e4876234d4e --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:16:16.159353", + "startedAt": "2025-01-08T01:16:15.732198", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3748.2597499999997, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4497.364, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.936, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 563.400520324707 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c22d0681d28810022fe5d3832f3f0bc756fa7eea --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "dd301aef3dc772a0c0e9a6fa4fc3085d943533957948129f00f5979b9ffd1681", "size": 723458, "path": "media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4"}, "global_step": 504, "_timestamp": 1736299243.213718, "_runtime": 267.4743149280548, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "77e85503de0c92b9ff4c6bb8563e3659b1e0fad00ababa5df43f9804ab53de51", "size": 596530, "path": "media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4"}, "eval/num_success": 3.0, "episode": 55.0, "eval/success_rate": 0.6000000238418579, "eval/success_rate_lb_0.01": 0.2069481611251831, "eval/success_rate_ub_0.01": 0.9527384638786316, "eval/success_rate_lb_0.025": 0.20744311809539795, "eval/success_rate_ub_0.025": 0.9364891648292542, "eval/success_rate_lb_0.05": 0.2632499635219574, "eval/success_rate_ub_0.05": 0.8492476344108582, "eval/success_rate_lb_0.1": 0.24815547466278076, "eval/success_rate_ub_0.1": 0.808523416519165, "eval/2_eval": {"_type": "video-file", "sha256": "2011ea908dfd24dcf4b7465954307a84c0f785a14213aa0faaf416172dc56302", "size": 1471675, "path": "media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c14c10349ec35d2de07af10177ac61ad953ef0509b67ec8b4202ff0aec7d5ede", "size": 1522101, "path": "media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "84c6290cf5d078bb7ea707febc1b18a7112acacf61f3aa0a667439b45013bd5f", "size": 1485602, "path": "media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "21a9f3410d4e4ea9386f8f8b05d066553e0d5981bed4f42ec4bc66fd0dcd2c14", "size": 401, "artifact_path": "wandb-client-artifact://s9yyy38v874ahgbdcig0eqwyg16rxbdqjr1vc2tcdpb032f0391nsqz8lczeahj1t9dll7drsvs9vwhd0xp9hogtul1urqkgt3tljr9p7ozz2h2sfnbzslpb33u4uxuh/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://umll6hizdmd36ye26kvv3h71oj8y4zaepz2stwaz4waxjq8cxofmve0vnh87cs1j7cc9w24k1l8si8l4isk48ipde751yr9cgdmwffzg92ham08m0vanq3hpe9c3pbwv:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 37, "total_num_episodes": 55, "total_success_rate": 0.6727272727272727, "total_success_rate_lb_0.01": 0.526706188911819, "total_success_rate_ub_0.01": 0.8014992176883421, "total_success_rate_lb_0.025": 0.5372565612122409, "total_success_rate_ub_0.025": 0.7817560433777918, "total_success_rate_lb_0.05": 0.5558311007805268, "total_success_rate_ub_0.05": 0.7704544662888351, "total_success_rate_lb_0.1": 0.5955363474595208, "total_success_rate_ub_0.1": 0.7525842189283649, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "bb62a15e6e7db2d4c602cc015a99ac2e25336ce7264994525c32b073b9fdcf41", "size": 519, "artifact_path": "wandb-client-artifact://zuhdxck4xrspbtqr1qc2uf3e5n9h1w5znfnfbmdf972vueqw89nizmcglxtx1ymf5univucwxxlieln9cenah38ggkenivg3p4tgzz7nfkioh0zoruucgeq1dh41dq7u/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://rwj0jynvrdrw2f94dp4okclyuubsxkx5g84noacz1bxa2hh6998q2u6uwf7i4d4tkuo6ugld4t4bm3mmeoh0k89nasja4a51zci1pfq6t8il6wq6z7b4ph2r9hh2oyla:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 270}, "eval/total_success_rate": 0.6727272868156433, "eval/total_success_rate_lb_0.01": 0.5267061591148376, "eval/total_success_rate_ub_0.01": 0.8014991879463196, "eval/total_success_rate_lb_0.025": 0.5372565388679504, "eval/total_success_rate_ub_0.025": 0.7817560434341431, "eval/total_success_rate_lb_0.05": 0.5558310747146606, "eval/total_success_rate_ub_0.05": 0.770454466342926, "eval/total_success_rate_lb_0.1": 0.5955363512039185, "eval/total_success_rate_ub_0.1": 0.7525842189788818} \ No newline at end of file diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug-internal.log b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..7cc3688a316349a64a920fb41f2b0ed9b4ad71ae --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug-internal.log @@ -0,0 +1,773 @@ +2025-01-07 20:16:15,738 INFO StreamThr :2222977 [internal.py:wandb_internal():85] W&B internal server running at pid: 2222977, started at: 2025-01-07 20:16:15.738368 +2025-01-07 20:16:15,740 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:16:15,754 INFO WriterThread:2222977 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/run-q50r8fwa.wandb +2025-01-07 20:16:15,754 DEBUG SenderThread:2222977 [sender.py:send():379] send: header +2025-01-07 20:16:15,761 DEBUG SenderThread:2222977 [sender.py:send():379] send: run +2025-01-07 20:16:16,079 INFO SenderThread:2222977 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files +2025-01-07 20:16:16,079 INFO SenderThread:2222977 [sender.py:_start_run_threads():1188] run started: q50r8fwa with start time 1736298975.739403 +2025-01-07 20:16:16,095 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:16:16,096 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:16:16,140 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:16:16,145 DEBUG HandlerThread:2222977 [system_info.py:__init__():26] System info init +2025-01-07 20:16:16,145 DEBUG HandlerThread:2222977 [system_info.py:__init__():41] System info init done +2025-01-07 20:16:16,145 INFO HandlerThread:2222977 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:16:16,145 INFO SystemMonitor:2222977 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:16:16,146 INFO HandlerThread:2222977 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:16:16,146 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:16:16,147 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:16:16,147 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:16:16,148 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:16:16,149 INFO SystemMonitor:2222977 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:16:16,159 DEBUG HandlerThread:2222977 [system_info.py:probe():152] Probing system +2025-01-07 20:16:16,160 DEBUG HandlerThread:2222977 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:16:16,165 DEBUG HandlerThread:2222977 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:16:16,165 DEBUG HandlerThread:2222977 [system_info.py:probe():200] Probing system done +2025-01-07 20:16:16,165 DEBUG HandlerThread:2222977 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:16:16.159353', 'startedAt': '2025-01-08T01:16:15.732198', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3748.2597499999997, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.364, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.936, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 563.400520324707}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:16:16,165 INFO HandlerThread:2222977 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:16:16,165 INFO HandlerThread:2222977 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:16:16,165 DEBUG HandlerThread:2222977 [system_info.py:_save_code():44] Saving code +2025-01-07 20:16:16,167 DEBUG HandlerThread:2222977 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:16:16,167 DEBUG HandlerThread:2222977 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:16:16,230 DEBUG HandlerThread:2222977 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:16:16,230 INFO HandlerThread:2222977 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:16:16,233 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:16:16,233 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:16:16,233 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:16:16,234 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:16:16,485 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:16:16,485 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:16:16,487 DEBUG SenderThread:2222977 [sender.py:send():379] send: telemetry +2025-01-07 20:16:16,500 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/ph1tuwzo-diff.patch +2025-01-07 20:16:16,501 INFO wandb-upload_0:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/wf4ttdkm-wandb-metadata.json +2025-01-07 20:16:16,517 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/5cyb1ogy-code/FISH/eval_robot.py +2025-01-07 20:16:16,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:16:16,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:16,529 INFO HandlerThread:2222977 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/tb" + save: true +} +control { + relay_id: "b561f5dd91de41acb6506c805f35aec4" +} +_info { + stream_id: "q50r8fwa" +} + +2025-01-07 20:16:16,529 DEBUG HandlerThread:2222977 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:16:16,531 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:16:16,623 DEBUG SenderThread:2222977 [sender.py:send():379] send: tbrecord +2025-01-07 20:16:16,623 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:16:16,623 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 with policy live +2025-01-07 20:16:16,804 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/xsx4ygmv-events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:16:17,081 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/diff.patch +2025-01-07 20:16:17,081 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:16:17,081 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:17,081 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/requirements.txt +2025-01-07 20:16:17,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-metadata.json +2025-01-07 20:16:17,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code/FISH/eval_robot.py +2025-01-07 20:16:17,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code/FISH +2025-01-07 20:16:17,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code +2025-01-07 20:16:17,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:18,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:19,080 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:19,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:20,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:21,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:21,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:22,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:23,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:23,929 DEBUG SenderThread:2222977 [sender.py:send():379] send: config +2025-01-07 20:16:24,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:25,082 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:25,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:26,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:26,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:27,083 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:27,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:28,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:29,083 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:29,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:30,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:31,084 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:31,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:16:31,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:16:31,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:31,605 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:32,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:33,085 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:33,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:34,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:35,085 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:35,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:36,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:36,851 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:37,086 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:37,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:38,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:39,086 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:39,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:40,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:41,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:42,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:42,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:43,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:44,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:45,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:46,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:16:46,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:16:46,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:47,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:47,615 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:48,089 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml +2025-01-07 20:16:48,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:49,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:50,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:51,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:52,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:52,747 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:53,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:53,713 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:16:53,714 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:16:53,714 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 with policy now +2025-01-07 20:16:53,714 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:16:53,714 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:16:53,716 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:16:53,966 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/2wxp72mb-media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 +2025-01-07 20:16:54,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 +2025-01-07 20:16:54,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:16:54,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media +2025-01-07 20:16:54,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos +2025-01-07 20:16:54,092 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:16:54,358 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/bbxcq12e-media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 +2025-01-07 20:16:54,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:55,091 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:55,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:56,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:57,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:58,349 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:16:58,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:16:59,093 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:16:59,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:00,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:01,093 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:01,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:17:01,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:17:01,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:02,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:03,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:03,633 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:04,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:05,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:06,095 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:17:06,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:07,095 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:07,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:08,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:09,096 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:09,136 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:09,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:10,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:11,097 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:11,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:12,494 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:13,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:14,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:14,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:15,497 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:16,149 DEBUG SystemMonitor:2222977 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:17:16,150 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:17:16,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:17:16,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:17:16,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:17,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:18,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:19,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:19,617 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:20,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:21,537 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:22,498 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:22,833 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:17:22,834 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:17:22,834 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 with policy now +2025-01-07 20:17:22,841 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:17:22,841 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:17:22,841 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:17:23,078 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/a0gwq0is-media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 +2025-01-07 20:17:23,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:17:23,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:23,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 +2025-01-07 20:17:23,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:17:23,354 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/cwj5csrf-media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 +2025-01-07 20:17:23,498 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:24,501 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:24,846 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:25,101 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:25,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:26,501 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:27,102 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:27,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:28,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:29,501 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:30,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:30,773 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:31,103 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:31,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:17:31,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:17:31,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:32,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:33,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:34,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:35,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:36,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:36,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:37,503 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:38,502 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:39,503 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:40,504 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:41,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:41,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:42,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:43,503 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:44,503 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:45,504 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:46,150 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:17:46,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:17:46,490 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:17:46,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:46,556 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:47,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:48,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:48,916 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:17:48,916 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:17:48,916 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:17:49,111 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:17:49,111 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:17:49,355 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/7uom1kkc-events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:17:49,525 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:50,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:51,112 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:51,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:52,197 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:52,505 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:53,113 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:53,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:54,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:55,114 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:17:55,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:56,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:57,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:17:57,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:58,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:17:59,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:00,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:01,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:18:01,486 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:18:01,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:02,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:02,554 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:03,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:04,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:05,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:06,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:07,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:08,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:08,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:09,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:10,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:11,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:12,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:13,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:14,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:14,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:15,508 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:16,151 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:18:16,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:18:16,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:18:16,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:17,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:18,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:19,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:19,582 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:20,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:21,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:22,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:23,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:24,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:25,491 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:25,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:26,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:27,510 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:28,512 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:29,124 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:29,521 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:29,543 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:18:29,543 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 with policy now +2025-01-07 20:18:29,585 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:18:29,588 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:18:29,588 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:18:29,596 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:18:29,802 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/52jaumdd-media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 +2025-01-07 20:18:30,125 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:18:30,125 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 +2025-01-07 20:18:30,125 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:18:30,418 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/o7mnof7h-media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 +2025-01-07 20:18:30,516 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:30,617 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:31,125 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:31,492 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:18:31,492 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:18:31,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:32,126 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:32,532 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:33,527 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:34,525 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:35,127 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:35,529 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:36,172 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:36,524 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:37,127 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:37,531 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:38,531 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:39,528 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:39,756 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:18:39,756 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:18:39,757 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:18:40,128 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:18:40,129 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:18:40,532 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:41,129 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:41,441 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:41,532 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:42,532 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:43,129 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:43,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:44,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:45,130 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:45,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:46,151 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:18:46,486 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:18:46,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:18:46,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:46,602 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:47,131 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:18:47,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:48,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:49,533 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:50,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:51,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:52,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:52,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:53,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:54,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:55,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:56,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:57,535 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:58,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:18:58,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:18:59,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:00,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:01,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:19:01,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:19:01,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:02,536 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:03,136 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:03,537 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:04,045 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:04,537 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:05,537 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:06,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:07,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:08,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:09,495 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:09,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:10,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:11,538 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:12,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:13,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:14,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:14,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:15,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:16,152 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:19:16,487 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:19:16,487 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:19:16,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:17,540 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:18,540 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:19,540 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:19,644 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:20,557 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:21,549 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:22,011 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:19:22,011 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 with policy now +2025-01-07 20:19:22,015 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:19:22,015 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:19:22,015 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:19:22,016 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:19:22,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:19:22,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 +2025-01-07 20:19:22,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:19:22,275 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/ocp1hzz3-media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 +2025-01-07 20:19:22,542 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:22,587 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/9dgm77nd-media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 +2025-01-07 20:19:23,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:23,546 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:24,144 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:24,558 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:25,058 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:25,554 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:26,552 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:27,145 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:27,556 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:28,553 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:29,552 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:30,558 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:30,613 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:31,147 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:31,493 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:19:31,493 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:19:31,558 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:32,559 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:33,557 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:34,557 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:35,556 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:36,514 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:36,557 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:37,558 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:38,559 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:39,564 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:40,565 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:41,566 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:42,509 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:42,576 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:43,151 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:19:43,393 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/awgb8vwr-events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:19:43,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:43,922 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:19:43,922 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:19:43,923 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:19:44,151 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:19:44,578 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:45,152 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:45,578 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:46,152 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:19:46,490 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:19:46,490 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:19:46,575 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:47,567 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:47,754 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:48,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:49,154 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:19:49,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:50,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:51,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:52,568 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:53,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:53,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:54,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:55,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:56,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:57,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:58,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:19:59,506 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:19:59,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:00,569 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:01,488 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:20:01,489 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:20:01,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:02,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:03,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:04,570 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:04,611 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:05,571 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:06,571 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:07,571 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:08,571 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:09,572 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:10,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:10,572 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:11,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:12,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:13,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:14,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:15,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:16,152 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:20:16,153 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:16,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:20:16,489 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:20:16,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:17,163 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:17,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:18,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:19,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:20,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:21,507 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:21,573 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:22,574 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:23,577 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:23,956 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:20:23,956 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 with policy now +2025-01-07 20:20:23,960 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:20:23,961 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:23,961 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:23,961 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:24,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:24,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 +2025-01-07 20:20:24,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval +2025-01-07 20:20:24,262 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/9n9nd5sa-media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 +2025-01-07 20:20:24,541 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/o9n3dcfm-media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 +2025-01-07 20:20:24,577 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:25,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:25,579 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:26,166 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:26,520 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:26,578 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:27,580 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:28,580 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:29,167 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:29,578 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:30,583 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:31,496 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:20:31,497 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:20:31,584 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:31,632 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:32,579 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:33,169 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:33,582 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:34,585 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:35,590 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:36,596 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:37,514 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:37,599 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:38,598 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:39,599 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:40,602 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:41,605 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:42,007 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:42,007 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:42,008 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:42,172 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:20:42,172 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:42,605 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:42,641 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:20:42,641 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:20:42,824 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:20:43,138 INFO wandb-upload_1:2222977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpian69z5u +2025-01-07 20:20:43,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:43,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-07 20:20:43,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media +2025-01-07 20:20:43,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval +2025-01-07 20:20:43,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table +2025-01-07 20:20:43,308 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:20:43,316 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:20:43,574 INFO SenderThread:2222977 [sender.py:send_request_log_artifact():1525] logged artifact run-q50r8fwa-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM3NTUzMw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyODk5Nw==', 'latestArtifact': None}} +2025-01-07 20:20:43,574 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:20:43,574 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json with policy now +2025-01-07 20:20:43,574 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:43,574 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:43,574 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,575 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,575 DEBUG SenderThread:2222977 [sender.py:send():379] send: telemetry +2025-01-07 20:20:43,575 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,579 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,579 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,580 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,580 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,580 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,580 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,581 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,581 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,581 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,581 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,582 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,582 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,582 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,582 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,583 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,583 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,583 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,583 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,583 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,584 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:43,584 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:43,584 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:20:43,601 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:43,761 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/cbz7vsyf-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-07 20:20:44,028 INFO wandb-upload_1:2222977 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpq1fkeuv9 +2025-01-07 20:20:44,093 INFO wandb-upload_0:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/2z7qu50a-media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-07 20:20:44,173 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:20:44,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:44,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json +2025-01-07 20:20:44,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval +2025-01-07 20:20:44,579 INFO SenderThread:2222977 [sender.py:send_request_log_artifact():1525] logged artifact run-q50r8fwa-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM3NTU1Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgyOTAwMg==', 'latestArtifact': None}} +2025-01-07 20:20:44,579 DEBUG SenderThread:2222977 [sender.py:send():379] send: files +2025-01-07 20:20:44,579 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json with policy now +2025-01-07 20:20:44,581 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:44,581 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:44,582 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:44,601 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:44,755 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/3ptywkim-media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json +2025-01-07 20:20:44,833 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/x9x0b433-events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:20:45,001 INFO wandb-upload_0:2222977 [upload_job.py:push():130] Uploaded file /tmp/tmpo7cfg653wandb/qrd4opyq-media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json +2025-01-07 20:20:45,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:45,174 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:45,602 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:46,153 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:20:46,489 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:20:46,489 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:20:46,621 DEBUG SenderThread:2222977 [sender.py:send():379] send: exit +2025-01-07 20:20:46,621 INFO SenderThread:2222977 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:20:46,621 INFO SenderThread:2222977 [sender.py:send_exit():588] handling runtime: 270 +2025-01-07 20:20:46,622 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:46,622 INFO SenderThread:2222977 [sender.py:send_exit():594] send defer +2025-01-07 20:20:46,622 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,622 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:20:46,622 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:46,623 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:20:46,623 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:20:46,623 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,623 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:20:46,623 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:46,623 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:20:46,623 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:20:46,623 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,623 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:20:46,623 INFO HandlerThread:2222977 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:20:46,623 DEBUG SystemMonitor:2222977 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:20:46,624 DEBUG SystemMonitor:2222977 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:20:46,625 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:20:46,625 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:20:46,676 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:20:46,676 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:20:46,677 INFO HandlerThread:2222977 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:20:46,677 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:46,677 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:20:46,677 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:20:46,677 DEBUG SenderThread:2222977 [sender.py:send():379] send: stats +2025-01-07 20:20:46,677 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,677 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:20:46,678 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:46,678 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:20:46,678 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:20:46,678 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:46,678 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:20:47,175 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:49,176 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:53,012 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:53,012 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:53,012 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:53,012 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:20:53,012 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:20:53,012 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:20:53,012 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:53,013 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:53,013 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:20:53,013 DEBUG SenderThread:2222977 [sender.py:send():379] send: history +2025-01-07 20:20:53,014 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:20:53,015 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:53,015 DEBUG SenderThread:2222977 [sender.py:send():379] send: summary +2025-01-07 20:20:53,016 INFO SenderThread:2222977 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:20:53,016 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:53,016 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:20:53,016 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:20:53,016 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:53,016 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:20:53,016 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:53,016 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:20:53,020 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:20:53,177 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:53,301 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:20:53,301 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:53,301 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:20:53,301 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:53,301 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:20:53,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:54,177 INFO Thread-12 :2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml +2025-01-07 20:20:55,559 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:20:55,559 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:55,559 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:55,560 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:20:55,560 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:55,560 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:20:55,560 INFO SenderThread:2222977 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:20:55,562 INFO SenderThread:2222977 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:20:55,564 INFO SenderThread:2222977 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:20:55,577 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:20:55,577 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:55,577 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:20:55,577 DEBUG SenderThread:2222977 [sender.py:send():379] send: artifact +2025-01-07 20:20:55,831 INFO SenderThread:2222977 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:20:55,831 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:55,831 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:20:55,831 INFO SenderThread:2222977 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:20:56,178 INFO SenderThread:2222977 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:56,179 INFO SenderThread:2222977 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files +2025-01-07 20:20:56,179 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml config.yaml +2025-01-07 20:20:56,179 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json wandb-summary.json +2025-01-07 20:20:56,179 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/requirements.txt requirements.txt +2025-01-07 20:20:56,182 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:20:56,182 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 events.out.tfevents.1736298976.leonmkim-ROG-Strix-G15CS-G15CS.2222778.0 +2025-01-07 20:20:56,182 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log output.log +2025-01-07 20:20:56,182 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/diff.patch diff.patch +2025-01-07 20:20:56,185 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 media/videos/eval/2_eval_2_2011ea908dfd24dcf4b7.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 media/videos/eval/0_eval_0_dd301aef3dc772a0c0e9.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 media/videos/eval/4_eval_4_84c6290cf5d078bb7ea7.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 media/videos/eval/1_eval_1_77e85503de0c92b9ff4c.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 media/videos/eval/3_eval_3_c14c10349ec35d2de07a.mp4 +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json media/table/eval/total_success_rate_ci_6_bb62a15e6e7db2d4c602.table.json +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json media/table/eval/success_rate_ci_5_21a9f3410d4e4ea9386f.table.json +2025-01-07 20:20:56,186 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:20:56,186 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,187 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:20:56,187 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,187 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:20:56,187 INFO SenderThread:2222977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:20:56,364 INFO wandb-upload_2:2222977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/wandb-summary.json +2025-01-07 20:20:56,367 INFO wandb-upload_0:2222977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/requirements.txt +2025-01-07 20:20:56,398 INFO wandb-upload_3:2222977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/output.log +2025-01-07 20:20:56,416 INFO wandb-upload_1:2222977 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/files/config.yaml +2025-01-07 20:20:56,539 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:56,540 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:56,617 INFO Thread-11 (_thread_body):2222977 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:20:56,617 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,617 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:20:56,617 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,617 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:20:56,617 INFO SenderThread:2222977 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:20:56,618 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:20:56,619 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,619 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:20:56,619 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,619 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:20:56,619 INFO SenderThread:2222977 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:20:56,806 INFO SenderThread:2222977 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:20:56,806 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:20:56,806 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,806 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:20:56,806 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,806 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:20:56,806 INFO SenderThread:2222977 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:20:56,807 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:20:56,807 DEBUG SenderThread:2222977 [sender.py:send():379] send: final +2025-01-07 20:20:56,807 INFO HandlerThread:2222977 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:20:56,807 DEBUG SenderThread:2222977 [sender.py:send():379] send: footer +2025-01-07 20:20:56,807 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: defer +2025-01-07 20:20:56,807 INFO SenderThread:2222977 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:20:56,807 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:56,808 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:56,808 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:20:56,808 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:20:56,808 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:20:56,808 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:20:56,810 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:20:56,811 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:20:56,855 DEBUG SenderThread:2222977 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:20:56,855 INFO MainThread:2222977 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:20:56,856 INFO MainThread:2222977 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:20:56,856 INFO MainThread:2222977 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:20:56,856 DEBUG HandlerThread:2222977 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:20:56,856 INFO HandlerThread:2222977 [handler.py:finish():882] shutting down handler +2025-01-07 20:20:57,808 INFO WriterThread:2222977 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/run-q50r8fwa.wandb +2025-01-07 20:20:57,855 INFO SenderThread:2222977 [sender.py:finish():1615] shutting down sender +2025-01-07 20:20:57,856 INFO SenderThread:2222977 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:20:57,856 INFO SenderThread:2222977 [file_pusher.py:join():175] waiting for file pusher diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug.log b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..8aeacb153ed9d328cde59f493ce86984c3bd597c --- /dev/null +++ b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Configure stats pid to 2222778 +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/settings +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug.log +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/logs/debug-internal.log +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa'} +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:init():619] starting backend +2025-01-07 20:16:15,733 INFO MainThread:2222778 [wandb_init.py:init():623] setting up manager +2025-01-07 20:16:15,737 INFO MainThread:2222778 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:16:15,739 INFO MainThread:2222778 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:16:15,753 INFO MainThread:2222778 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:16:15,759 INFO MainThread:2222778 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:16:16,095 INFO MainThread:2222778 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:16:16,136 INFO MainThread:2222778 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:16:16,136 INFO MainThread:2222778 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:16:16,486 INFO MainThread:2222778 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:16:16,486 INFO MainThread:2222778 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:16:16,486 INFO MainThread:2222778 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:16:16,486 INFO MainThread:2222778 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:16:16,489 INFO MainThread:2222778 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:16:16,490 INFO MainThread:2222778 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/q50r8fwa/tb, True +2025-01-07 20:16:23,929 INFO MainThread:2222778 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:20:57,858 WARNING MsgRouterThr:2222778 [router.py:message_loop():77] message_loop has been closed diff --git a/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/run-q50r8fwa.wandb b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/run-q50r8fwa.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9a9d21a4ce7ec7ed7894c42248576a4bd2454ac4 Binary files /dev/null and b/q50r8fwa/wandb/run-20250107_201615-q50r8fwa/run-q50r8fwa.wandb differ diff --git a/snapshot_10541.pt b/snapshot_10541.pt new file mode 100644 index 0000000000000000000000000000000000000000..d9276c79ff926478ba1c9c8ceceeae676d815df4 --- /dev/null +++ b/snapshot_10541.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33fe6ab83625cbcc1e7cd05d202d6a62457c940e238b72dae21947f1a8c5afdf +size 910446022 diff --git a/snapshot_11499.pt b/snapshot_11499.pt new file mode 100644 index 0000000000000000000000000000000000000000..2f2a07c801688e396baf6c35321e7c31c76c712e --- /dev/null +++ b/snapshot_11499.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1569f341343da46d1cbc7c67564e3b5ac26562332844fcc7903205ccd5a66ae +size 910446022 diff --git a/snapshot_12000.pt b/snapshot_12000.pt new file mode 100644 index 0000000000000000000000000000000000000000..fe6728f7b0963f5e6a37142d180cc737e83349c1 --- /dev/null +++ b/snapshot_12000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4dbf8438abfbeed0cfd784e9bda1b299d8533173a0bbf8e4a00f9129e7470a14 +size 910446022 diff --git a/snapshot_12009.pt b/snapshot_12009.pt new file mode 100644 index 0000000000000000000000000000000000000000..8b9a4b00387b6bbc0b5e2ca582c0db02d70feeb8 --- /dev/null +++ b/snapshot_12009.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0485011d16f9953bf668297d2d99cd26a378e0c185ce1f8a52e2c1babb3dc2d +size 910446022 diff --git a/snapshot_12907.pt b/snapshot_12907.pt new file mode 100644 index 0000000000000000000000000000000000000000..78ba699dd3a27870d7f7fe03e73575c7a3883a46 --- /dev/null +++ b/snapshot_12907.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d5f6e85275d3f6e21931e6896caecc55b773c2d4c66554bda3fcb885bc6e3c49 +size 910446022 diff --git a/snapshot_14348.pt b/snapshot_14348.pt new file mode 100644 index 0000000000000000000000000000000000000000..c7a3eb3b50365e741b00b37e152e406daacee98e --- /dev/null +++ b/snapshot_14348.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:049e8238bb454869bc9d65b0275e6aa6c4a52a27da14a2154a9c1bf63ac98706 +size 910446022 diff --git a/snapshot_4249.pt b/snapshot_4249.pt new file mode 100644 index 0000000000000000000000000000000000000000..8558a885c1ec36cfc9c5a6626bbfe8a85916426a --- /dev/null +++ b/snapshot_4249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4096286259fa5e20e30d6b73191182b0273120d4fa8e270994c5d769c2e9ecc +size 910446022 diff --git a/snapshot_4500.pt b/snapshot_4500.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa6ad9b3285f05cfc1e407c4e43239bfac03b193 --- /dev/null +++ b/snapshot_4500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0975ea14ee4b870cf339ba1410ca26b0ce1f1a025195dadf646739f8256ab0d1 +size 910446022 diff --git a/snapshot_6691.pt b/snapshot_6691.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c8d29a902fa2112e2e63fa17281672f85aa4757 --- /dev/null +++ b/snapshot_6691.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f1acc0320cc0a4bb806cc9715c56c69d828f20a1e063b4c8515a6796eef4ca0b +size 910446022 diff --git a/snapshot_7249.pt b/snapshot_7249.pt new file mode 100644 index 0000000000000000000000000000000000000000..8c7ec8b874b7953b87a1285d5d7c36952f3d8923 --- /dev/null +++ b/snapshot_7249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:882fd26a16db1ee02dc4d6ed6e85dc61845e3c14d3b21864cb1aeff22a86f4bb +size 910446022 diff --git a/snapshot_8504.pt b/snapshot_8504.pt new file mode 100644 index 0000000000000000000000000000000000000000..ed4edaad7f513fb48ba8f7a778f308513a4e012c --- /dev/null +++ b/snapshot_8504.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d01e33c0d6caa2908e78cf6804a71ec74c2fbeeafcdd19271cd4a934c4b41f19 +size 910446022 diff --git a/snapshot_9000.pt b/snapshot_9000.pt new file mode 100644 index 0000000000000000000000000000000000000000..9e7be859456d52a17854290376b23cb52b34554c --- /dev/null +++ b/snapshot_9000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a686389b08ecf1ac8e53b9e99a4fa8785b0e603067a7dcc8768cb531a5aa5398 +size 910446022 diff --git a/snapshot_9249.pt b/snapshot_9249.pt new file mode 100644 index 0000000000000000000000000000000000000000..201e8824d874a95642dbbfb2467fd0039e3dfc41 --- /dev/null +++ b/snapshot_9249.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de9b5fe9fbc5bb6c0ec7a89ebfd645b3e74aa6cc1b6bc858bf8d1a3d5de757fd +size 910446022 diff --git a/w4b02c7s/.hydra/config.yaml b/w4b02c7s/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3 --- /dev/null +++ b/w4b02c7s/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1013_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/w4b02c7s/.hydra/hydra.yaml b/w4b02c7s/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fac0c5c1081f934546ffcbf70a9713a7893600e9 --- /dev/null +++ b/w4b02c7s/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_robot + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/w4b02c7s/.hydra/overrides.yaml b/w4b02c7s/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/w4b02c7s/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/w4b02c7s/episode_rosbags/aligned_depth_to_color_K.npy b/w4b02c7s/episode_rosbags/aligned_depth_to_color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/w4b02c7s/episode_rosbags/aligned_depth_to_color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/w4b02c7s/episode_rosbags/cam_tf_world.npy b/w4b02c7s/episode_rosbags/cam_tf_world.npy new file mode 100644 index 0000000000000000000000000000000000000000..d1402b0af577e3ba43b0861d252985014316f40a --- /dev/null +++ b/w4b02c7s/episode_rosbags/cam_tf_world.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29313ba240dd65ebbc79056d5582a97c9586cf2a1d4a1e0db13b87b49152cc9e +size 256 diff --git a/w4b02c7s/episode_rosbags/color_K.npy b/w4b02c7s/episode_rosbags/color_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..6948db2888eafe8e3ce6f490d46d96055c1c0604 --- /dev/null +++ b/w4b02c7s/episode_rosbags/color_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a962c703d20da282f5e009d432dff51df4ebd22f3386699b6754ea9cfc06a55 +size 200 diff --git a/w4b02c7s/episode_rosbags/depth_K.npy b/w4b02c7s/episode_rosbags/depth_K.npy new file mode 100644 index 0000000000000000000000000000000000000000..deed9ab1575ca77fba97e7fb863162972c3896ce --- /dev/null +++ b/w4b02c7s/episode_rosbags/depth_K.npy @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc601d2fecd31c5513c76a88b5d4d8059adc1f92dad682e2d755d89a66d8fdf7 +size 200 diff --git a/w4b02c7s/episode_rosbags/episode_0_2025-01-07-20-21-32.bag b/w4b02c7s/episode_rosbags/episode_0_2025-01-07-20-21-32.bag new file mode 100644 index 0000000000000000000000000000000000000000..b1117ec05221db33629b09330945c54a055a3f34 --- /dev/null +++ b/w4b02c7s/episode_rosbags/episode_0_2025-01-07-20-21-32.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50699f7ef01773ae41f02329f413e55529fb452d02af26161432cb81ff9bda69 +size 925836015 diff --git a/w4b02c7s/episode_rosbags/episode_1_2025-01-07-20-22-04.bag b/w4b02c7s/episode_rosbags/episode_1_2025-01-07-20-22-04.bag new file mode 100644 index 0000000000000000000000000000000000000000..f799d19e97c87066f708f7d3f6832fbffd667065 --- /dev/null +++ b/w4b02c7s/episode_rosbags/episode_1_2025-01-07-20-22-04.bag @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cde3425cfa182f6fe4604bccc6a553f4c2892d9a4f1e712b858fba712c95930 +size 2604662927 diff --git a/w4b02c7s/eval_robot.log b/w4b02c7s/eval_robot.log new file mode 100644 index 0000000000000000000000000000000000000000..7f4fc327a299c91dafe1b44b9f3009f3bf035da8 --- /dev/null +++ b/w4b02c7s/eval_robot.log @@ -0,0 +1,12 @@ +[2025-01-07 20:21:08,324][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:596: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-07 20:21:08,327][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_robot.py:532: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + diff --git a/w4b02c7s/eval_video/0_eval.mp4 b/w4b02c7s/eval_video/0_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df24aca29fa62f1c8757a288469847e61cbd75a7 --- /dev/null +++ b/w4b02c7s/eval_video/0_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bec6c78d6d5d8da97ef5b12f58d06b35eefbf3e99b7c219ecd57f38aa6f9592e +size 598726 diff --git a/w4b02c7s/eval_video/1_eval.mp4 b/w4b02c7s/eval_video/1_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8c7ce920b562c296f1be71a9696713da7e269f40 --- /dev/null +++ b/w4b02c7s/eval_video/1_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:407b26ce0b9569f5bbfddb38fc8f461c936cbaa4ff10b329dd8eafdd5f9b0b29 +size 1442639 diff --git a/w4b02c7s/eval_video/2_eval.mp4 b/w4b02c7s/eval_video/2_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b598eb16287c92cffd128dcdb3591adccdad80f0 --- /dev/null +++ b/w4b02c7s/eval_video/2_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a036985f900b09e578db306820019b257b10dfa7e10ec7292e161c4a7ddcf402 +size 1446625 diff --git a/w4b02c7s/eval_video/3_eval.mp4 b/w4b02c7s/eval_video/3_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e428eb281eb61855609aabc8374dea175c6eb355 --- /dev/null +++ b/w4b02c7s/eval_video/3_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5460bdfba046140b658a20b3293bd3947a1f5cb63ddde4af2adb234ed176831 +size 1444170 diff --git a/w4b02c7s/eval_video/4_eval.mp4 b/w4b02c7s/eval_video/4_eval.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a560d50d88dfd4c7eac168e209ce3ecf5be502b --- /dev/null +++ b/w4b02c7s/eval_video/4_eval.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc542bb6f251b5d41e0ca2a9c9bb7811e9ed3945d0d3e9e1fe3d72db8478a6a9 +size 1448685 diff --git a/w4b02c7s/tb/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 b/w4b02c7s/tb/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 new file mode 100644 index 0000000000000000000000000000000000000000..a9745b94f8e36531b5df28b42249277ab3f5d20c --- /dev/null +++ b/w4b02c7s/tb/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7858bce89f41e8e297f39d235976c57c39bfe1a146d370aca655ef1a10d37665 +size 6519 diff --git a/w4b02c7s/wandb/debug-internal.log b/w4b02c7s/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..98388bc48b22895665214d8f786742a6294e3d3f --- /dev/null +++ b/w4b02c7s/wandb/debug-internal.log @@ -0,0 +1,789 @@ +2025-01-07 20:21:14,147 INFO StreamThr :2224087 [internal.py:wandb_internal():85] W&B internal server running at pid: 2224087, started at: 2025-01-07 20:21:14.146861 +2025-01-07 20:21:14,149 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:21:14,160 INFO WriterThread:2224087 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/run-w4b02c7s.wandb +2025-01-07 20:21:14,161 DEBUG SenderThread:2224087 [sender.py:send():379] send: header +2025-01-07 20:21:14,168 DEBUG SenderThread:2224087 [sender.py:send():379] send: run +2025-01-07 20:21:14,462 INFO SenderThread:2224087 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files +2025-01-07 20:21:14,462 INFO SenderThread:2224087 [sender.py:_start_run_threads():1188] run started: w4b02c7s with start time 1736299274.148413 +2025-01-07 20:21:14,479 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:21:14,480 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:21:14,555 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:21:14,560 DEBUG HandlerThread:2224087 [system_info.py:__init__():26] System info init +2025-01-07 20:21:14,560 DEBUG HandlerThread:2224087 [system_info.py:__init__():41] System info init done +2025-01-07 20:21:14,560 INFO HandlerThread:2224087 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:21:14,561 INFO SystemMonitor:2224087 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:21:14,561 INFO HandlerThread:2224087 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:21:14,561 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:21:14,562 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:21:14,563 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:21:14,563 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:21:14,564 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:21:14,573 DEBUG HandlerThread:2224087 [system_info.py:probe():152] Probing system +2025-01-07 20:21:14,575 DEBUG HandlerThread:2224087 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:21:14,581 DEBUG HandlerThread:2224087 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:21:14,581 DEBUG HandlerThread:2224087 [system_info.py:probe():200] Probing system done +2025-01-07 20:21:14,581 DEBUG HandlerThread:2224087 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:21:14.573756', 'startedAt': '2025-01-08T01:21:14.136932', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.56425, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4494.112, 'min': 800.0, 'max': 4700.0}, {'current': 4499.827, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 572.616024017334}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:21:14,581 INFO HandlerThread:2224087 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:21:14,581 INFO HandlerThread:2224087 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:21:14,581 DEBUG HandlerThread:2224087 [system_info.py:_save_code():44] Saving code +2025-01-07 20:21:14,584 DEBUG HandlerThread:2224087 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:21:14,584 DEBUG HandlerThread:2224087 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:21:14,650 DEBUG HandlerThread:2224087 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:21:14,651 INFO HandlerThread:2224087 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:21:14,654 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:21:14,654 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:21:14,654 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:21:14,654 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:21:14,876 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/1svrxmfh-wandb-metadata.json +2025-01-07 20:21:14,891 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:21:14,891 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:21:14,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:14,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:21:14,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:21:14,907 INFO wandb-upload_2:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/yhj813vc-diff.patch +2025-01-07 20:21:14,927 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/w5iryu7n-code/FISH/eval_robot.py +2025-01-07 20:21:14,933 INFO HandlerThread:2224087 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/tb" + save: true +} +control { + relay_id: "426b523202ad4154bb07363de3b5e08c" +} +_info { + stream_id: "w4b02c7s" +} + +2025-01-07 20:21:14,933 DEBUG HandlerThread:2224087 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:21:14,945 DEBUG SenderThread:2224087 [sender.py:send():379] send: telemetry +2025-01-07 20:21:14,998 DEBUG SenderThread:2224087 [sender.py:send():379] send: tbrecord +2025-01-07 20:21:14,999 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:21:14,999 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 with policy live +2025-01-07 20:21:15,173 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/wscof4s7-events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-metadata.json +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/requirements.txt +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code/FISH/eval_robot.py +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/diff.patch +2025-01-07 20:21:15,465 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:15,465 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code +2025-01-07 20:21:15,465 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code/FISH +2025-01-07 20:21:15,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:16,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:17,463 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:17,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:18,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:19,350 DEBUG SenderThread:2224087 [sender.py:send():379] send: config +2025-01-07 20:21:19,350 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:19,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:20,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:21,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:21,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:22,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:23,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:24,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:24,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:25,465 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:25,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:26,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:27,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:28,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:29,467 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:29,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:21:29,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:21:29,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:30,036 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:30,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:31,467 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:31,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:32,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:33,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:34,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:35,468 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:35,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:35,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:36,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:37,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:38,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:39,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:40,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:40,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:41,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:42,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:43,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:44,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:21:44,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:21:44,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:45,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:46,395 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:46,615 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:21:46,616 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 with policy now +2025-01-07 20:21:46,616 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:21:46,617 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:21:46,617 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:21:46,619 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:21:46,851 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/brgutril-media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 +2025-01-07 20:21:46,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:21:47,743 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/aawlbvhc-media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 +2025-01-07 20:21:47,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:48,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:49,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:49,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:50,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:51,477 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:51,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:51,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:52,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:53,478 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:53,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:54,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:55,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:56,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:57,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:57,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:58,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:59,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:21:59,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:21:59,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:00,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:01,480 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:01,480 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:22:01,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:02,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:03,481 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:03,585 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:03,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:04,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:05,482 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:05,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:06,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:07,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:08,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:08,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:09,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:10,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:11,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:12,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:13,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:13,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:14,564 DEBUG SystemMonitor:2224087 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:22:14,564 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:22:14,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:22:14,894 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:22:14,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:15,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:16,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:17,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:18,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:19,841 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:19,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:20,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:21,487 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:21,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:22,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:23,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:24,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:24,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:25,489 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:25,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:26,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:27,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:28,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:29,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:22:29,894 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:22:29,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:30,045 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:30,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:31,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:32,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:33,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:34,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:35,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:35,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:36,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:37,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:38,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:39,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:40,245 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:22:40,248 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:22:40,248 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 with policy now +2025-01-07 20:22:40,248 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:22:40,248 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:22:40,249 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:22:40,494 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:22:40,494 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 +2025-01-07 20:22:40,495 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:22:40,520 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/i8mv5ivj-media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 +2025-01-07 20:22:40,806 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/nn13y66a-media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 +2025-01-07 20:22:40,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:41,249 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:41,495 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:41,904 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:42,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:43,906 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:44,565 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:22:44,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:22:44,901 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:22:44,945 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:45,496 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:45,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:46,669 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:46,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:47,904 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:48,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:49,498 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:49,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:50,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:51,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:51,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:52,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:53,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:54,338 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:22:54,338 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:22:54,339 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:22:54,499 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:22:54,499 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:22:54,739 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/0yikza4i-events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:22:54,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:55,500 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:55,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:56,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:56,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:57,500 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:57,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:58,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:59,501 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:59,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:22:59,896 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:22:59,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:00,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:01,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:02,003 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:02,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:03,503 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:03,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:04,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:05,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:06,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:07,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:07,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:08,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:09,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:10,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:11,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:12,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:12,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:13,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:14,565 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:23:14,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:23:14,896 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:23:14,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:15,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:16,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:17,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:18,003 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:18,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:19,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:20,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:21,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:22,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:23,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:23,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:24,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:25,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:26,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:27,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:28,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:28,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:29,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:23:29,897 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:23:29,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:30,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:31,913 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:32,921 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:33,458 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:23:33,458 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 with policy now +2025-01-07 20:23:33,471 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:23:33,472 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:23:33,472 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:23:33,476 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:23:33,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:33,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:23:33,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 +2025-01-07 20:23:33,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:23:33,709 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/v51k45zd-media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 +2025-01-07 20:23:33,914 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:33,990 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/js1noyat-media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 +2025-01-07 20:23:34,511 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:34,916 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:35,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:35,921 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:36,925 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:37,922 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:38,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:39,515 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:39,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:39,924 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:40,925 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:41,516 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:41,929 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:42,928 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:43,927 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:44,566 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:23:44,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:23:44,901 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:23:44,945 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:44,982 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:45,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:46,941 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:47,931 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:48,182 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:23:48,182 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:23:48,184 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:23:48,518 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:23:48,518 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:23:48,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:49,518 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:49,944 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:50,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:50,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:51,519 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:51,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:52,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:53,520 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:53,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:54,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:55,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:56,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:56,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:57,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:58,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:59,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:23:59,897 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:23:59,941 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:00,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:01,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:02,018 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:02,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:03,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:04,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:05,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:06,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:07,245 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:07,524 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:07,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:08,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:09,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:10,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:11,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:12,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:12,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:13,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:14,566 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:24:14,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:24:14,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:24:14,941 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:15,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:16,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:17,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:17,958 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:18,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:19,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:20,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:21,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:22,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:23,302 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:23,530 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:23,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:24,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:25,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:26,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:27,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:27,880 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:24:27,880 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 with policy now +2025-01-07 20:24:27,888 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:24:27,888 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:24:27,888 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:24:27,895 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:24:27,942 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:28,173 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/xg728rec-media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 +2025-01-07 20:24:28,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:24:28,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 +2025-01-07 20:24:28,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:24:28,833 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/20hvtfdl-media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 +2025-01-07 20:24:28,915 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:28,944 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:29,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:29,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:24:29,899 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:24:29,943 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:30,957 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:31,954 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:32,951 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:33,534 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:33,953 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:34,418 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:34,958 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:35,534 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:35,958 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:36,953 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:37,960 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:38,964 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:39,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:39,967 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:40,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:41,963 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:42,537 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:24:42,815 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/zr5gfofa-events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:24:42,971 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:43,247 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:24:43,247 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:24:43,247 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:24:43,537 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:43,537 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:24:43,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:44,567 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:24:44,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:24:44,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:24:44,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:45,043 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:45,538 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:45,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:46,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:47,539 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:47,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:48,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:49,540 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:49,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:50,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:50,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:51,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:52,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:53,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:54,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:55,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:56,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:56,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:57,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:58,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:59,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:24:59,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:24:59,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:00,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:01,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:01,971 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:02,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:03,967 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:04,967 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:05,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:06,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:07,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:07,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:08,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:09,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:10,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:11,547 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:11,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:12,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:13,017 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:13,548 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:13,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:14,567 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:25:14,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:25:14,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:25:14,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:15,549 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:15,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:16,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:17,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:18,050 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:18,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:19,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:20,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:21,974 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:22,780 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:25:22,780 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 with policy now +2025-01-07 20:25:22,781 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:25:22,782 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:25:22,782 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:22,801 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:22,973 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:23,069 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/28l1v0g1-media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 +2025-01-07 20:25:23,551 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:23,551 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:25:23,552 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 +2025-01-07 20:25:23,552 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:25:23,807 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:23,880 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/q1iq46wv-media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 +2025-01-07 20:25:23,976 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:24,981 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:25,552 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:25,982 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:27,001 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:27,553 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:27,987 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:28,980 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:29,321 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:29,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:25:29,908 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:25:29,979 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:30,982 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:31,554 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:31,984 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:32,992 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:33,992 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:34,922 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:34,995 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:36,002 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:37,002 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:37,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:38,072 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:25:38,072 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:25:38,230 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:25:38,522 INFO wandb-upload_1:2224087 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprnm7pcgi +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table +2025-01-07 20:25:38,663 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:25:38,676 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:25:38,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:39,075 INFO SenderThread:2224087 [sender.py:send_request_log_artifact():1525] logged artifact run-w4b02c7s-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM4Mzc5MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzMTIwMg==', 'latestArtifact': None}} +2025-01-07 20:25:39,075 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:25:39,075 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json with policy now +2025-01-07 20:25:39,075 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:25:39,075 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,076 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,076 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:25:39,076 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,080 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,080 DEBUG SenderThread:2224087 [sender.py:send():379] send: telemetry +2025-01-07 20:25:39,080 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,081 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,081 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,081 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,081 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,082 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,082 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,082 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,082 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,083 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,083 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,083 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,083 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,083 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,084 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,084 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,084 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,084 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,085 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,085 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,085 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,085 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,085 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:25:39,251 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/tuf9qeia-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-07 20:25:39,481 INFO wandb-upload_0:2224087 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqcd0wrrf +2025-01-07 20:25:39,503 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/d6bqe7eu-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-07 20:25:39,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:39,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:25:39,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:25:39,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json +2025-01-07 20:25:39,559 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval +2025-01-07 20:25:39,863 INFO SenderThread:2224087 [sender.py:send_request_log_artifact():1525] logged artifact run-w4b02c7s-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM4MzgxNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzMTIwMw==', 'latestArtifact': None}} +2025-01-07 20:25:39,863 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:25:39,863 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json with policy now +2025-01-07 20:25:39,865 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:25:39,865 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,866 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:40,049 INFO wandb-upload_2:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/8mliszsk-media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json +2025-01-07 20:25:40,306 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/9ys5wxb3-media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json +2025-01-07 20:25:40,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:25:40,916 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:40,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:41,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:41,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:42,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:43,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:44,568 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:25:44,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:25:44,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:25:44,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:45,978 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:45,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:46,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:47,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:48,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:49,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:50,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:51,916 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:51,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:52,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:53,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:54,889 DEBUG SenderThread:2224087 [sender.py:send():379] send: exit +2025-01-07 20:25:54,889 INFO SenderThread:2224087 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:25:54,889 INFO SenderThread:2224087 [sender.py:send_exit():588] handling runtime: 280 +2025-01-07 20:25:54,890 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:54,890 INFO SenderThread:2224087 [sender.py:send_exit():594] send defer +2025-01-07 20:25:54,890 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,890 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:25:54,890 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:25:54,890 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:25:54,890 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:25:54,890 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,890 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:25:54,891 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:25:54,891 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:25:54,891 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:25:54,891 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,891 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:25:54,891 INFO HandlerThread:2224087 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:25:54,891 DEBUG SystemMonitor:2224087 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:25:54,891 DEBUG SystemMonitor:2224087 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:25:54,892 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:25:54,892 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:25:54,945 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:25:54,945 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:25:54,945 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:25:54,946 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:25:54,946 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:25:54,946 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:25:54,946 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:25:54,946 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,946 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:25:54,946 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:25:54,946 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:25:54,946 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:25:54,947 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,947 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:25:55,565 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:25:57,566 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:26:01,320 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:01,321 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:01,321 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:26:01,321 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:01,322 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:26:01,322 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:26:01,322 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:01,322 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:01,322 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:26:01,323 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:26:01,323 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:26:01,323 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:26:01,324 DEBUG SenderThread:2224087 [sender.py:send():379] send: summary +2025-01-07 20:26:01,327 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:26:01,328 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:01,328 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:26:01,328 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:26:01,328 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:01,328 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:26:01,328 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:01,328 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:26:01,331 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:01,445 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:26:01,445 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:01,445 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:26:01,445 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:01,446 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:26:01,567 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml +2025-01-07 20:26:01,567 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:26:01,890 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:04,759 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:26:04,759 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:04,760 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:04,760 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:26:04,760 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:04,760 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:26:04,760 INFO SenderThread:2224087 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:26:04,762 INFO SenderThread:2224087 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:26:04,764 INFO SenderThread:2224087 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:26:04,779 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:26:04,779 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:04,779 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:26:04,779 DEBUG SenderThread:2224087 [sender.py:send():379] send: artifact +2025-01-07 20:26:04,891 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:04,993 INFO SenderThread:2224087 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:26:04,993 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:04,993 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:26:04,993 INFO SenderThread:2224087 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:26:05,568 INFO SenderThread:2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml config.yaml +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json wandb-summary.json +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/requirements.txt requirements.txt +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log output.log +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/diff.patch diff.patch +2025-01-07 20:26:05,570 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 +2025-01-07 20:26:05,575 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json +2025-01-07 20:26:05,575 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-07 20:26:05,575 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:26:05,575 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:05,576 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:05,578 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:26:05,578 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:05,578 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:26:05,578 INFO SenderThread:2224087 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:26:05,753 INFO wandb-upload_2:2224087 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml +2025-01-07 20:26:05,808 INFO wandb-upload_3:2224087 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/requirements.txt +2025-01-07 20:26:05,808 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:26:05,814 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/dnrqfq78-events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:26:05,820 INFO wandb-upload_4:2224087 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:26:05,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:05,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:06,021 INFO Thread-11 (_thread_body):2224087 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:26:06,021 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:06,021 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:26:06,021 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:06,021 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:26:06,021 INFO SenderThread:2224087 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:26:06,023 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:26:06,023 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:06,023 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:26:06,023 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:06,023 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:26:06,023 INFO SenderThread:2224087 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:26:06,091 INFO SenderThread:2224087 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:26:06,091 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:26:06,091 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:06,091 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:26:06,091 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:06,091 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:26:06,091 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:26:06,091 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:06,092 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:26:06,092 DEBUG SenderThread:2224087 [sender.py:send():379] send: final +2025-01-07 20:26:06,092 DEBUG SenderThread:2224087 [sender.py:send():379] send: footer +2025-01-07 20:26:06,092 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:06,092 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:26:06,092 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:06,093 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:06,093 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:26:06,093 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:06,093 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:26:06,094 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:26:06,094 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:26:06,096 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:06,144 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:06,144 INFO MainThread:2224087 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:26:06,144 INFO MainThread:2224087 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:26:06,145 INFO MainThread:2224087 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:26:06,145 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:26:06,145 INFO HandlerThread:2224087 [handler.py:finish():882] shutting down handler +2025-01-07 20:26:07,094 INFO WriterThread:2224087 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/run-w4b02c7s.wandb +2025-01-07 20:26:07,144 INFO SenderThread:2224087 [sender.py:finish():1615] shutting down sender +2025-01-07 20:26:07,144 INFO SenderThread:2224087 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:26:07,144 INFO SenderThread:2224087 [file_pusher.py:join():175] waiting for file pusher diff --git a/w4b02c7s/wandb/debug.log b/w4b02c7s/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6795a233759d3fbc5adb02cc4d2bcf59e06be7af --- /dev/null +++ b/w4b02c7s/wandb/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:21:14,138 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:21:14,138 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Configure stats pid to 2223909 +2025-01-07 20:21:14,138 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:21:14,138 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/settings +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug.log +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug-internal.log +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s'} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:init():619] starting backend +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:init():623] setting up manager +2025-01-07 20:21:14,146 INFO MainThread:2223909 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:21:14,147 INFO MainThread:2223909 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:21:14,159 INFO MainThread:2223909 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:21:14,166 INFO MainThread:2223909 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:21:14,479 INFO MainThread:2223909 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:21:14,549 INFO MainThread:2223909 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:21:14,550 INFO MainThread:2223909 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:21:14,892 INFO MainThread:2223909 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:21:14,892 INFO MainThread:2223909 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:21:14,892 INFO MainThread:2223909 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:21:14,892 INFO MainThread:2223909 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:21:14,893 INFO MainThread:2223909 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:21:14,894 INFO MainThread:2223909 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/tb, True +2025-01-07 20:21:19,349 INFO MainThread:2223909 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:26:07,146 WARNING MsgRouterThr:2223909 [router.py:message_loop():77] message_loop has been closed diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code/FISH/eval_robot.py b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code/FISH/eval_robot.py new file mode 100644 index 0000000000000000000000000000000000000000..50493fea183960d2ea4fa0b904b2e6077c21a710 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code/FISH/eval_robot.py @@ -0,0 +1,605 @@ +#%% +import warnings +import os + +os.environ['MKL_SERVICE_FORCE_INTEL'] = '1' +os.environ['MUJOCO_GL'] = 'egl' +from pathlib import Path +#%% +import hydra +import numpy as np +import torch + +import utils +from utils import get_feature_dirname_from_configs + +from video import VideoRecorder +import pickle +import time +import threading +import shutil +from logger import Logger + +import wandb +from omegaconf import OmegaConf, open_dict + +from replay_buffer_robot import RosbagEvalReplayBufferStorage +from lerobot.common.utils.utils import _relative_path_between + +torch.backends.cudnn.benchmark = True +warnings.filterwarnings('ignore', category=DeprecationWarning) + +# import specs for replay buffer +from dm_env import specs + +import sys, signal +import yaml + +import binomial_cis as bc + +# get path of current file +current_path = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(current_path, os.pardir)) +# from contact_estimation.src.utils.viz_utils import normalized_surface_normal_to_rgb, depth_map_to_im, grasped_env_dtc_map_to_im, contact_prob_map_to_im, desaturate_color_image, masked_overlay_im_list + +def make_agent(obs_spec, action_spec, cfg): + cfg.obs_shape = obs_spec['pixels'].shape + dataset_statistics = None # this will be loaded from the checkpoint + try: + cfg.action_shape = action_spec.shape + except: + pass + return hydra.utils.instantiate(cfg, dataset_statistics) + +class Workspace: + def __init__(self, cfg): + self.work_dir = Path.cwd() + print(f'workspace: {self.work_dir}') + + signal.signal(signal.SIGINT, self.signal_handler) + + self.cfg = cfg + self.loading_uncompiled_checkpoint_with_compile = False + self.loading_compiled_checkpoint_with_no_compile = False + + snapshot_path = Path(self.cfg.checkpoint_weight_dir) / f'snapshot_{self.cfg.checkpoint_epoch}.pt' + self.load_checkpoint_conf(snapshot_path=snapshot_path) + + # load config for action trajectories + utils.set_seed_everywhere(self.cfg.seed) + self.device = torch.device(self.cfg.device) + self.setup() + + # self.agent = make_agent(self.eval_env.observation_spec(), + # self.eval_env.action_spec(), self.cfg.agent) + self.timer = utils.Timer() + # self._global_step = 0 + self._global_episode = 0 + self._global_epoch = 0 + self.num_episode_successes = 0 + + self.alpha_range = [.01, .025, .05, .1] + + # Need to convert hydra config to primitive container for wandb https://docs.wandb.ai/guides/integrations/hydra + with open_dict(self.cfg): + self.cfg.feature_type = get_feature_dirname_from_configs( + hydra.utils.instantiate(self.cfg.agent.config.observation_cfg), + self.cfg.agent.config.policy_cfg.input_shapes, + hydra.utils.instantiate(self.cfg.agent.config.policy_cfg.action_history_encoder_config) if 'observation.action_history' in self.cfg.agent.config.policy_cfg.input_shapes else None, + ) + + wandb_config = OmegaConf.to_container( + self.cfg, resolve=True, throw_on_missing=True + ) + # must be called before any tf summary writer is created + if self.cfg.use_wandb: + # get the run id from the final_experiment_dir directory + run_id = os.path.basename(os.path.normpath(self.cfg.final_experiment_dir)) + wandb.init(project='extrinsic_contact_downstream', entity='serialexperimentsleon', job_type='eval', sync_tensorboard=self.cfg.use_tb, config=wandb_config, id=run_id) + + self.logger = Logger(self.work_dir, use_tb=self.cfg.use_tb, use_wandb=self.cfg.use_wandb) + + # if not self.loading_uncompiled_checkpoint_with_compile and self.cfg.agent.config.compile: + # self.agent.compile_modules() + + # self.load_checkpoint(snapshot_path=snapshot_path) + + # if self.loading_uncompiled_checkpoint_with_compile: # need to call compile after loading the checkpoint + # self.agent.compile_modules() + + print(f"loaded agent with feature_type: {self.cfg.feature_type}") + + def check_for_key_press(self): + while self.continue_keypress_thread: + inp = input("Press 'r' to restart current episode, 'n' to stop current episode and skip to next, 'q' to break entire eval\n") + if inp == 'n': + self.preempt_episode = True + print("preempting episode") + elif inp in ['', '0', '1']: # enter key + if inp in ['0', '1']: + self.num_episode_successes += int(inp) + self.proceed_after_env_reset_event.set() + print("proceeding to start episode!") + elif inp == 'q': + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + print("quitting eval") + break + elif inp == 'r': + print('restarting episode') + self.preempt_episode = True + self.restart_episode = True + else: + print("Invalid key press, try again") + + # self.keypress_input_thread.join() # wait for the keypress thread to finish + + def signal_handler(self, signal, frame): + print("\nprogram exiting gracefully") + self.proceed_after_env_reset_event.set() + self.preempt_episode = True + self.exit_eval = True + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + video_filepath = self.video_recorder.save() + # get the video file and convert to video tensor to log + self.logger.log_video('eval/video', video_filepath, self.global_step) + wandb.finish() + sys.exit(0) + + def setup(self): + # create envs + self.eval_env = hydra.utils.call(self.cfg.suite.task_make_fn) + # expert_demo_config_path = os.path.join(os.path.dirname(self.cfg.expert_dataset), 'demo_config.yaml') + # self.expert_demo_config = yaml.load(open(expert_demo_config_path, 'r'), Loader=yaml.FullLoader) + # self.eval_env._env.action_trans_norm = expert_demo_config['max_translation_action_norm'] + # self.eval_env._env.action_rot_norm = expert_demo_config['max_rotation_action_norm'] + # self.eval_env._env.action_period = expert_demo_config['sample_period'] + # print(f"setting max_translation_action_norm to {expert_demo_config['max_translation_action_norm']} and sample_period to {expert_demo_config['sample_period']}") + # print(f"setting max_rotation_action_norm to {expert_demo_config['max_rotation_action_norm']}") + + # self.eval_env.set_demo_params(self.cfg.expert_dataset) + + # Turn off random start + self.eval_env.random_start = False + + # create replay buffer + # data_specs = [ + # { + # 'observation': self.eval_env.observation_spec(), + # }, + # # self.eval_env.observation_spec()['features'], + # self.eval_env.action_spec(), + # specs.Array(self.eval_env.action_spec().shape, self.eval_env.action_spec().dtype, 'vinn_action'), + # specs.Array((1, ), np.float32, 'reward'), + # specs.Array((1, ), np.float32, 'discount'), + # ] + + # self.eval_replay_storage = ZarrEvalReplayBufferStorage(data_specs, self.work_dir / 'eval_buffer', debug_timestamps=self.cfg.debug_timestamps, save_buffer=self.cfg.save_buffer, debug_info_data_specs=self.eval_env.debug_info_data_specs, camera_info_dict=self.eval_env.get_camera_info_dict()) + self.eval_replay_storage = RosbagEvalReplayBufferStorage(self.work_dir) + + self.video_recorder = VideoRecorder( + self.work_dir if self.cfg.save_video else None, + ros_enabled=True, + fps=self.cfg.agent.config.policy_frequency, + ) + + print('workspace setup complete') + + @property + def global_step(self): + # return self._global_step + return self.eval_env.get_global_step() + + @property + def global_episode(self): + return self._global_episode + + @property + def global_frame(self): + return self.global_step * self.cfg.action_repeat + + @property + def global_epoch(self): + return self._global_epoch + + def reset(self, eval_idx): + if not self.eval_env.enable_arm: + return np.array([0,0,0], dtype=np.float32) + self.eval_env.arm_refresh(reset=False) + # Set start position + try: + self.eval_env.set_position(self.start_pos[eval_idx]) + except: + self.eval_env.arm.set_position(self.start_pos[eval_idx]) + if self.eval_env.arm.keep_gripper_closed: + self.eval_env.arm.close_gripper_fully() + else: + self.eval_env.arm.open_gripper_fully() + time.sleep(0.1) + time_step = self.eval_env.step(np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32), + np.zeros(self.eval_env.action_spec().shape[0], dtype=np.float32)) + return time_step + + def eval(self): + # before evals start, prompt user for name of grasped object and the left book of the slot location + grasped_obj_name = input("Enter the name of the grasped object: ") + left_book_slot = input("Enter the left book slot location: ") + # update wandb config + if self.cfg.use_wandb: + wandb.config.update({'grasped_obj_name': grasped_obj_name, 'left_book_slot': left_book_slot}) + + self.preempt_episode = False + self.exit_eval = False + self.restart_episode = False + + self.continue_keypress_thread = True + self.proceed_after_env_reset_event = threading.Event() + self.keypress_input_thread = threading.Thread(target=self.check_for_key_press) + self.keypress_input_thread.start() + + # # Set model to eval mode + # self.agent.train(False) + + eval_until_episode = utils.Until(self.cfg.num_eval) + + self.use_action_history = False + # if "dp" in repr(self.agent) and "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + if "observation.action_history" in self.cfg.agent.config.policy_cfg.input_shapes: + self.use_action_history = True + + # self.eval_replay_storage._new_eval_step(0) + + # if 'vinn' in repr(self.agent) or 'openloop' in repr(self.agent): + # with open(self.cfg.expert_dataset, 'rb') as f: + # if self.cfg.obs_type == 'pixels': + # self.expert_demo, _, self.expert_action, self.expert_reward = pickle.load(f) + # elif self.cfg.obs_type == 'features': + # _, self.expert_demo, self.expert_action, self.expert_reward = pickle.load(f) + + # if self.cfg.action_trajectories: + # with open(self.cfg.expert_action_trajectories, 'rb') as f: + # self.expert_action = pickle.load(f) + + # if isinstance(self.cfg.train_demo_idxs_list_or_num, int): + # if self.cfg.train_demo_idxs_list_or_num == -1: + # self.cfg.train_demo_idxs_list_or_num = len(self.expert_demo) + # train_demo_idxs_list_or_num = list(range(self.cfg.train_demo_idxs_list_or_num)) + + # self.expert_demo = self.expert_demo[train_demo_idxs_list_or_num] + # self.expert_action = self.expert_action[train_demo_idxs_list_or_num] + # self.expert_reward = self.expert_reward[train_demo_idxs_list_or_num] + # # if self.cfg.action_plans: + # # self.expert_action_plans = self.expert_action_plans[self.cfg.train_demo_idxs_list_or_num] + # # self.expert_demo = self.expert_demo[:self.cfg.num_demos] + # # self.expert_action = self.expert_action[:self.cfg.num_demos] + # # self.expert_reward = self.expert_reward[:self.cfg.num_demos] + + # self.expert_demo = np.concatenate(self.expert_demo, axis=0) + # self.expert_rgb_obs = np.ascontiguousarray(np.transpose(self.expert_demo, (0,2,3,1))[:, :,:,:3].astype(np.uint8)) + # self.expert_action = np.concatenate(self.expert_action, axis=0) + + # self.agent.save_representations(self.expert_demo, self.expert_action, 128, config=self.expert_demo_config) + + # Get start points + if self.cfg.random_start: + eval_starts = Path(self.cfg.eval_starts) / 'starts.pkl' + if eval_starts.exists(): + with eval_starts.open('rb') as f: + self.start_pos = pickle.load(f) + else: + eval_starts = Path(self.cfg.eval_starts) + eval_starts.mkdir(parents=True, exist_ok=True) + + # Generate start points + self.start_pos = [] + try: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.get_random_pos()) + except: + for _ in range(self.cfg.num_eval): + self.start_pos.append(self.eval_env.arm.get_random_pos()) + + # Save start points for the task + eval_starts = eval_starts / 'starts.pkl' + with eval_starts.open('wb') as f: + pickle.dump(self.start_pos, f) + + time_step = self.eval_env.reset() + # replay_thread = None + while eval_until_episode(self.global_episode) and not self.exit_eval: + # self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + print(f"Starting episode {self.global_episode}") + time_step = self.eval_env.reset() #Leon: need to call reset twice in case objects are trapped + self.video_recorder.init(self.eval_env, video_filename=f'{self.global_episode}_eval.mp4') + # x = input("Press Enter to continue... after reseting env") + print("Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = [] + table_data = [] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + time_step = self.eval_env.reset() + # debug_info_dict = self.eval_env.debug_info_dict + # if replay_thread is not None: + # # wait for the last replay thread to finish + # replay_thread.join() + + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + + # replay_thread.start() + if self.cfg.random_start: + time_step = self.reset(self.global_episode) + time.sleep(2) #5) + # if 'vinn' in repr(self.agent): + # self.agent.reset() + # # self.agent.buffer.reset() + # # if self.cfg.open_loop: + # # self.agent.current_step = 0 + # if 'openloop' in repr(self.agent): + # self.agent.curr_step = 0 + # at start of each episode, provide zero action for policies that use action history + # shape should be (T_o, T_a, action_dim) + + # while not time_step.last() and not self.preempt_episode: + self.video_recorder.ros_start_recording() + self.eval_replay_storage.start_episode() + self.eval_env.start_policy_timer() + while not self.eval_env.episode_done() and not self.preempt_episode: + # with torch.no_grad(), utils.eval_mode(self.agent): + # # if self.cfg.agent.provide_topk: + # # action, vinn_action, topk = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # elif self.cfg.agent.provide_obs: + # # action, vinn_action, obs = self.agent.act( + # # time_step.observation['pixels'], + # # self.global_step, + # # eval_mode=True) + # # else: + # action, vinn_action = self.agent.act( + # time_step.observation, + # self.global_step, + # eval_mode=True, + # obs_timestamp=time_step.observation['timestamp'], + # obs_seq=time_step.observation['seq'], + # action_history=action_history, + # action_history_start_timestamp=action_history_start_timestamp, + # ) + # DONT WAIT FOR POLICY TO GET AN ACTION + # we dont want to slow down grabbing obs and passing to sam/contact features + + self.eval_env.run_policy_threads() # this just does a rospy sleep + + # if self.use_action_history: + # action_history_start_timestamp = time_step.observation['timestamp'] + # # action_history = action[:self.cfg.agent.config.policy_cfg.action_history_encoder_config.history_length, ...] + # # add n_obs_steps dimension to action_history, for now we assume n_obs_steps = 1 + # # TODO: handle n_obs_steps > 1 + # action_history = action[np.newaxis, ...] + + # time_step = self.eval_env.step(action, vinn_action) # obs, reward after action has been taken + # debug_info_dict = self.eval_env.debug_info_dict + + # time_step = self.eval_env.ros_step() + + # replay_thread.join() + + # time how long it takes to execute the step + # time_before_add = time.perf_counter() + # self.eval_replay_storage.add(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict) + # use thread to call the add function in a separate thread + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step._replace(observation=time_step.observation[self.cfg.obs_type]), debug_info_dict)) + + # replay_thread = threading.Thread(target=self.eval_replay_storage.add, args=(time_step, debug_info_dict)) + # replay_thread.start() + + # print(f"Time to add to replay buffer: {time.perf_counter() - time_before_add}") + + # self.video_recorder.record(self.eval_env) + # self._global_step += 1 + + self.eval_env.stop_policy_timer() + + if self.restart_episode: + # means we should delete the current episode and start again + self.restart_episode = False + self.eval_replay_storage.reset_current_episode() + self.video_recorder.reset_current_episode() + + else: + self.eval_replay_storage.store_current_episode() + video_filepath = self.video_recorder.save() + self.logger.log_video(f"eval/{video_filepath.name.rstrip('.mp4')}", video_filepath, self.global_step) + self._global_episode += 1 + + self.preempt_episode = False # reset preempt_episode flag + + # self.video_recorder.save(f'{episode}_eval.mp4') + # get the video file and convert to video tensor to log + + self.eval_env.reset() + + print("Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success") + self.proceed_after_env_reset_event.clear() # clear the event flag + self.proceed_after_env_reset_event.wait() # blocking wait for the event flag to be set + if self.global_episode > 0: + # self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'num_success': self.num_episode_successes}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({'success_rate': self.num_episode_successes/self.global_episode}, self.global_step, 'eval', episode=self.global_episode) + + # log confidence intervals for success rate + k = self.num_episode_successes # number of successes + n = self.global_episode # number of trials + + table_columns = ['success_rate'] + table_data = [self.num_episode_successes/self.global_episode] + for alpha in self.alpha_range: + lb = bc.binom_ci(k, n, alpha, 'lb') + ub = bc.binom_ci(k, n, alpha, 'ub') + + self.logger.log_metrics({f'success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=self.global_episode) + self.logger.log_metrics({f'success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=self.global_episode) + + table_columns.extend([f'success_rate_lb_{alpha}', f'success_rate_ub_{alpha}']) + table_data.extend([lb, ub]) + + table_data = [table_data] + + # seperately log as a table + wandb.log({ + "eval/success_rate_ci": wandb.Table(data=table_data, columns=table_columns) + }) + + # also accumulate eval metrics across previous eval runs + # TODO: change wandb init to resume from an existing run!!! + run_filter={ + "jobType": "eval", + "config.wandb_run_id": self.cfg.wandb_run_id, + "summary_metrics.episode": {"$gte": 5}, + "config.checkpoint_epoch": self.cfg.checkpoint_epoch, + "state": "finished", + # "config.grasped_obj_name": grasped_obj_name, + # "config.left_book_slot": left_book_slot, + } + + api = wandb.Api() + filtered_runs = api.runs("serialexperimentsleon/extrinsic_contact_downstream", filters=run_filter) + total_num_successes = self.num_episode_successes + total_num_episodes = self.global_episode + list_of_historical_run_ids = [] + if len(filtered_runs) > 0: + for filtered_run in filtered_runs: + total_num_successes += filtered_run.summary_metrics['eval/num_success'] + # total_num_episodes += filtered_run.summary_metrics['episode'] + total_num_episodes += filtered_run.config['num_eval'] + list_of_historical_run_ids.append(filtered_run.id) + + wandb.summary['total_num_successes'] = total_num_successes + wandb.summary['total_num_episodes'] = total_num_episodes + wandb.summary['total_success_rate'] = total_num_successes/total_num_episodes + + # log the accumulated metrics as a table + total_table_columns = ['total_num_successes', 'total_num_episodes', 'total_success_rate'] + total_table_data = [total_num_successes, total_num_episodes, total_num_successes/total_num_episodes] + self.logger.log_metrics({'total_success_rate': total_num_successes/total_num_episodes}, self.global_step, 'eval', episode=total_num_episodes) + + for alpha in self.alpha_range: + lb = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'lb') + ub = bc.binom_ci(total_num_successes, total_num_episodes, alpha, 'ub') + total_table_columns.extend([f'total_success_rate_lb_{alpha}', f'total_success_rate_ub_{alpha}']) + total_table_data.extend([lb, ub]) + wandb.summary[f'total_success_rate_lb_{alpha}'] = lb + wandb.summary[f'total_success_rate_ub_{alpha}'] = ub + + self.logger.log_metrics({f'total_success_rate_lb_{alpha}': lb}, self.global_step, 'eval', episode=total_num_episodes) + self.logger.log_metrics({f'total_success_rate_ub_{alpha}': ub}, self.global_step, 'eval', episode=total_num_episodes) + + total_table_data = [total_table_data] + wandb.log({ + 'eval/total_success_rate_ci': wandb.Table(data=total_table_data, columns=total_table_columns) + }) + + self.continue_keypress_thread = False # will stop the keypress thread + self.keypress_input_thread.join() # wait for the keypress thread to finish + + def load_checkpoint_conf(self, snapshot_path): + config_path = snapshot_path.parent / 'config.yaml' + if not config_path.exists(): + raise FileNotFoundError(f'No snapshot conf found at {config_path}') + else: + # load the omegaconf config + hydra.core.global_hydra.GlobalHydra.instance().clear() + hydra.initialize( + str(_relative_path_between(Path(config_path).absolute().parent, Path(__file__).absolute().parent)), + ) + cfg = hydra.compose(Path(config_path).stem) + from deepdiff import DeepDiff + from omegaconf import open_dict + diff = DeepDiff(OmegaConf.to_container(cfg), OmegaConf.to_container(self.cfg)) # old, new + # import re + overwriteable_keys = [f"root{overwritable_key}" for overwritable_key in ["['use_wandb']", "['path_to_depth_extrinsics']", "['eval']", "['root_dir']", "['wandb_notes']", "['agent']['config']['train_cfg']['use_amp']", "['agent']['config']['compile']", "['agent']['config']['policy_cfg']['num_inference_steps']"]] + if "values_changed" in diff: + # top_k_checkpoints, wandb_notes, agent.config.train_cfg.use_amp, save_snapshot_every_epochs_diffusion, check_topk_every_epochs_diffusion, validate_diffusion_on_action_loss_every_epochs, train_eval_diffusion_on_action_loss_every_epochs, validate_every_epochs_diffusion + # for keys above, overwrite the old config with the new config + for k, v in diff['values_changed'].items(): + # replace any keys that are under "root['suite']" + if k in overwriteable_keys or k.startswith("root['suite']"): + print(f"Found changed key {k} with value {v}. Overwriting old checkpoint config") + if k == "root['agent']['config']['compile']": + if diff['values_changed'][k]['new_value']: + self.loading_uncompiled_checkpoint_with_compile = True + elif not diff['values_changed'][k]['new_value']: + # raise ValueError("Cannot load a compiled checkpoint without compile") + self.loading_compiled_checkpoint_with_no_compile = True + exec(f"{k.replace('root[', 'cfg[')} = {k.replace('root[', 'self.cfg[')}") + # for any new values, update the old checkpoint config + if "dictionary_item_added" in diff: + for new_key in diff['dictionary_item_added']: # this is a list + # if new_key == "root['suite']['task_make_fn']['observation_cfg']": + if new_key == "root['suite']['task_make_fn']['agent_policy_cfg']": + # pass the agents observation_cfg to the suite task_make_fn + with open_dict(cfg): # to allow addition of non-existing keys + # cfg.suite.task_make_fn.observation_cfg = cfg.agent.config.observation_cfg + cfg.suite.task_make_fn.agent_policy_cfg = cfg.agent.config + continue + elif "['agent']['config']['policy_cfg']['input_shapes']" in new_key: + # skip adding the new key if it is the input_shapes of the policy_cfg + continue + else: + print(f"Found new key {new_key} with value {eval(new_key.replace('root[', 'self.cfg['))}. Adding to checkpoint config") + # eval(new_key.replace('root', 'cfg')) = eval(new_key.replace('root', 'self.cfg')) + if new_key == "root['agent']['config']['compile']": + if self.cfg.agent.config.compile: + self.loading_uncompiled_checkpoint_with_compile = True + + with open_dict(cfg): + exec(f"{new_key.replace('root[', 'cfg[')}={new_key.replace('root[', 'self.cfg[')}") + self.cfg = cfg + + def load_checkpoint(self, snapshot_path, bc=False): + print(f'resuming {repr(self.agent)}: {snapshot_path}') + with snapshot_path.open('rb') as f: + payload = torch.load(f) + agent_payload = {} + for k, v in payload.items(): + if k not in self.__dict__: + agent_payload[k] = v + elif k == '_global_epoch': + self._global_epoch = v + print(f'loaded epoch: {v}') + if self.cfg.use_wandb: + # add to config of wandb + wandb.config.update({'epoch': v}) + + # self.agent.load_snapshot_eval(agent_payload, bc) + +@hydra.main(config_path='cfgs', config_name='config_eval') +def main(cfg): + from eval_robot import Workspace as W + root_dir = Path.cwd() + workspace = W(cfg) + + workspace.eval() + +if __name__ == '__main__': + main() diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3fe3565d27f2527099154fdf17d722bc8410c7b7 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml @@ -0,0 +1,895 @@ +wandb_version: 1 + +root_dir: + desc: null + value: /home/leonmkim/fish_leon +replay_buffer_size: + desc: null + value: 150000 +replay_buffer_num_workers: + desc: null + value: 2 +nstep: + desc: null + value: 3 +batch_size: + desc: null + value: 128 +seed: + desc: null + value: 1 +dataset_shuffle_seed: + desc: null + value: 0 +device: + desc: null + value: cuda +save_video: + desc: null + value: true +save_train_video: + desc: null + value: true +use_tb: + desc: null + value: true +use_wandb: + desc: null + value: true +wandb_run_id: + desc: null + value: '1013_1' +wandb_notes: + desc: null + value: 1013_1_restarted_11 +eval: + desc: null + value: true +true_action_history: + desc: null + value: false +train_pad_after: + desc: null + value: 4 +process_contact_features: + desc: null + value: true +obs_type: + desc: null + value: pixels +use_color: + desc: null + value: true +use_depth: + desc: null + value: true +use_masks: + desc: null + value: true +mask_list: + desc: null + value: + - EE_obj_mask +mask_representation: + desc: null + value: channels +crop_hw: + desc: null + value: + - 144 + - 144 +crop_down_offset: + desc: null + value: 48 +color_crop_type: + desc: null + value: null +depth_crop_type: + desc: null + value: null +segmask_crop_type: + desc: null + value: null +add_crop_binary_mask: + desc: null + value: false +add_coord_conv_map: + desc: null + value: false +use_context_color: + desc: null + value: false +use_context_depth: + desc: null + value: false +use_context_segmask: + desc: null + value: false +context_color_crop_type: + desc: null + value: null +context_depth_crop_type: + desc: null + value: null +context_segmask_crop_type: + desc: null + value: null +context_add_crop_binary_mask: + desc: null + value: false +context_add_coord_conv_map: + desc: null + value: false +use_contact_map: + desc: null + value: true +use_sdf_maps: + desc: null + value: true +use_normals_maps: + desc: null + value: true +which_objects: + desc: null + value: both +max_contact_prob: + desc: null + value: 0.1 +max_depth: + desc: null + value: 2.0 +grasped_dtc_max_value: + desc: null + value: 0.2 +env_dtc_max_value: + desc: null + value: 0.4 +grasped_normals_mask_max_dtc_value: + desc: null + value: 0.2 +env_normals_mask_max_dtc_value: + desc: null + value: 0.4 +clamp_dtc: + desc: null + value: true +dtc_adaptive_normalization: + desc: null + value: false +mask_normals_within_sdf: + desc: null + value: true +adaptive_normals_mask: + desc: null + value: true +learnable_contact_preprocess_params: + desc: null + value: true +contact_model_name: + desc: null + value: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 +contact_estimation_model_ckpt_path: + desc: null + value: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +encoder_type: + desc: null + value: small +debug_timestamps: + desc: null + value: false +open_loop: + desc: null + value: false +action_trajectories: + desc: null + value: true +stop_after_action: + desc: null + value: false +interpolation_frequency: + desc: null + value: 25 +policy_frequency: + desc: null + value: 5 +wait_for_new_camera_frames: + desc: null + value: true +baseline: + desc: null + value: false +train_demo_idxs_list_or_num: + desc: null + value: -1 +log_train_every_steps: + desc: null + value: 25 +name_of_expert_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +store_dataset_in_memory: + desc: null + value: false +expert_dataset: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +action_key: + desc: null + value: action_trajectory_25hz +semantic_demo_grouping_name: + desc: null + value: semantic_demo_grouping.yaml +semantic_demo_grouping: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml +include_groups_list: + desc: null + value: all +expert_dataset_config: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml +name_of_valid_demo: + desc: null + value: 120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +valid_dataset_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr +valid_demo_idxs_list_or_num: + desc: null + value: null +val_num_groups: + desc: null + value: 5 +load_bc: + desc: null + value: true +checkpoint_epoch_list: + desc: null + value: + - 99 + - 199 + - 299 + - 399 + - 499 + - 599 + - 699 + - 799 + - 899 + - 999 + - 1249 + - 1499 + - 1749 + - 1999 + - 2999 + - 3999 + - 4999 + - 5999 + - 6999 + - 7999 + - 8999 + - 9999 +snapshot_root_dir: + desc: null + value: /mnt/grasp_high_usage/leonmkim/contact_estimation/FISH +save_snapshot: + desc: null + value: true +save_last_snapshot: + desc: null + value: true +save_snapshot_when_done: + desc: null + value: true +top_k_checkpoints: + desc: null + value: 5 +save_snapshot_link_to_weights_dir: + desc: null + value: deprecated +bc_regularize: + desc: null + value: false +bc_weight_type: + desc: null + value: qfilter +experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +agent: + desc: null + value: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: true + device: cuda + n_obs_steps: 1 + suite_name: frankagym + obs_type: pixels + enable_arm: true + enable_camera: true + use_tb: true + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false +suite: + desc: null + value: + suite: frankagym + name: frankagym + frame_stack: 1 + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: FrankaInsertion-v1 + height: 240 + width: 320 + frame_stack: 1 + action_repeat: 1 + seed: 1 + enable_arm: true + enable_gripper: true + start_with_gripper_open: true + enable_camera: true + path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy + contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt + x_limit: + - 0.2 + - 0.7 + y_limit: + - -0.4 + - 0.4 + z_limit: + - -0.05 + - 0.55 + device: cuda + interpolation_frequency: 25 + policy_frequency: 5 + debug_timestamps: false + stop_after_action: false + open_loop: false + wait_for_new_camera_frames: true + action_key: action_trajectory_25hz + action_trajectory_horizon: 36 + action_trajectories: true + path_to_zarr_dataset: /home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr + agent_policy_cfg: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: cuda + cam_resize_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + policy_frequency: 5 + interpolation_frequency: 25 + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: 36 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: 6 + kernel_size: 5 + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + input_shapes: + observation.image: + - 13 + - 180 + - 240 + context_observation.image: + - 13 + - 180 + - 240 + observation.state: + - 8 + observation.action_history: + - 7 + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: 1000000 + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: true + use_color: true + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: true + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: false + use_depth: false + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: false + representation: channels + mask_list: + - EE_obj_mask + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: null + depth_crop_type: null + segmask_crop_type: null + crop_hw: + - 144 + - 144 + crop_down_offset: 48 + add_crop_binary_mask: false + add_coord_conv_map: false + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: cosine + use_contact_map: true + use_sdf_maps: true + use_normals_maps: true + which_objects: both + grasped_dtc_max_value: 0.2 + env_dtc_max_value: 0.4 + grasped_normals_mask_max_dtc_value: 0.2 + env_normals_mask_max_dtc_value: 0.4 + clamp_dtc: true + max_contact_prob: 0.1 + mask_normals_within_sdf: true + dtc_adaptive_normalization: false + adaptive_normals_mask: true + max_depth: 2.0 + image_shape: + - 13 + - 180 + - 240 + learnable_contact_preprocess_params: true + learning_rate: 0.0001 + weight_decay: 0.0 + contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9 + zero_centered: false + true_action_history: false +num_train_frames_bc: + desc: null + value: 50000 +num_train_frames_drq: + desc: null + value: 1100000 +stddev_schedule_drq: + desc: null + value: linear(1.0,0.1,100000) +task_name: + desc: null + value: FrankaInsertion-v1 +num_train_frames_vinn: + desc: null + value: 25000 +num_train_frames_diffusion: + desc: null + value: 1000000 +num_train_epochs_bc: + desc: null + value: 5000 +num_train_epochs_diffusion: + desc: null + value: 15000 +validate_every_epochs_bc: + desc: null + value: 5 +validate_every_epochs_diffusion: + desc: null + value: 250 +validate_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +train_eval_diffusion_on_action_loss_every_epochs: + desc: null + value: 250 +check_topk_every_epochs: + desc: null + value: 5 +check_topk_every_epochs_diffusion: + desc: null + value: 250 +save_snapshot_every_epochs_diffusion: + desc: null + value: 1500 +x_limit: + desc: null + value: + - 0.2 + - 0.7 +y_limit: + desc: null + value: + - -0.4 + - 0.4 +z_limit: + desc: null + value: + - -0.05 + - 0.55 +home_displacement: + desc: null + value: + - 0.55 + - 0.0 + - 0.55 + - 180.0 + - 0.0 + - 0.0 +enable_gripper: + desc: null + value: true +start_with_gripper_open: + desc: null + value: true +offset_mask: + desc: null + value: + - 1 + - 1 + - 1 + - 1 + - 1 + - 1 +path_to_depth_extrinsics: + desc: null + value: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy +feature_type: + desc: null + value: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +save_buffer: + desc: null + value: true +num_eval: + desc: null + value: 5 +random_start: + desc: null + value: false +eval_starts: + desc: null + value: /home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1 +num_valid_demos: + desc: null + value: null +load_checkpoint: + desc: null + value: true +checkpoint_epoch: + desc: null + value: 12000 +load_residual_weight: + desc: null + value: false +checkpoint_root_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH +checkpoint_weight_dir: + desc: null + value: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1 +residual_weight: + desc: null + value: /home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt +final_experiment_dir: + desc: null + value: ./exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s +_wandb: + desc: null + value: + code_path: code/FISH/eval_robot.py + python_version: 3.10.14 + cli_version: 0.17.5 + framework: torch + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1736299274 + t: + 1: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 2: + - 1 + - 41 + - 49 + - 50 + - 55 + - 83 + 3: + - 14 + - 16 + - 23 + - 35 + - 62 + 4: 3.10.14 + 5: 0.17.5 + 8: + - 5 + 13: linux-x86_64 +grasped_obj_name: + desc: null + value: modelsys +left_book_slot: + desc: null + value: twodim diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/diff.patch b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/diff.patch new file mode 100644 index 0000000000000000000000000000000000000000..8adda0290b36965f42f4119e05fd13ea9507158f --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/diff.patch @@ -0,0 +1,15 @@ +diff --git a/FISH/cfgs/config_eval.yaml b/FISH/cfgs/config_eval.yaml +index a5e82a4..1c0be7a 100644 +--- a/FISH/cfgs/config_eval.yaml ++++ b/FISH/cfgs/config_eval.yaml +@@ -139,8 +139,8 @@ load_checkpoint: ${agent.load_checkpoint} + + # RGBD+mask+contact(all ftrs)+act history + # wandb_run_id: '1003_0' +-# wandb_run_id: '1013_1' # seed 1 +-wandb_run_id: '1017_0' # dataset shuffle seed 1 ++wandb_run_id: '1013_1' # seed 1 ++# wandb_run_id: '1017_0' # dataset shuffle seed 1 + + # all books, 6/20 demos per book + # RGBD+mask+act history diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json new file mode 100644 index 0000000000000000000000000000000000000000..a2db2594aac2dfee94d7430f5fc4332216ce7fe1 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json @@ -0,0 +1 @@ +{"columns": ["success_rate", "success_rate_lb_0.01", "success_rate_ub_0.01", "success_rate_lb_0.025", "success_rate_ub_0.025", "success_rate_lb_0.05", "success_rate_ub_0.05", "success_rate_lb_0.1", "success_rate_ub_0.1"], "data": [[0.8, 0.37485200169205224, 0.9957208297155283, 0.3695220351480134, 0.9926368444172731, 0.4414144158480513, 0.954582236616294, 0.4180309623643781, 0.934476241382488]]} \ No newline at end of file diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json new file mode 100644 index 0000000000000000000000000000000000000000..9fa52ec681f6e410b284f866241e101fcc3f3b7b --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json @@ -0,0 +1 @@ +{"columns": ["total_num_successes", "total_num_episodes", "total_success_rate", "total_success_rate_lb_0.01", "total_success_rate_ub_0.01", "total_success_rate_lb_0.025", "total_success_rate_ub_0.025", "total_success_rate_lb_0.05", "total_success_rate_ub_0.05", "total_success_rate_lb_0.1", "total_success_rate_ub_0.1"], "data": [[41, 60, 0.6833333333333333, 0.5438502281816522, 0.8056211470946375, 0.5544648468385485, 0.7868945374511087, 0.5722976997350102, 0.7759805619164684, 0.6096675358493338, 0.7589508182782594]]} \ No newline at end of file diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..df24aca29fa62f1c8757a288469847e61cbd75a7 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bec6c78d6d5d8da97ef5b12f58d06b35eefbf3e99b7c219ecd57f38aa6f9592e +size 598726 diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..8c7ce920b562c296f1be71a9696713da7e269f40 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:407b26ce0b9569f5bbfddb38fc8f461c936cbaa4ff10b329dd8eafdd5f9b0b29 +size 1442639 diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..b598eb16287c92cffd128dcdb3591adccdad80f0 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a036985f900b09e578db306820019b257b10dfa7e10ec7292e161c4a7ddcf402 +size 1446625 diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e428eb281eb61855609aabc8374dea175c6eb355 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c5460bdfba046140b658a20b3293bd3947a1f5cb63ddde4af2adb234ed176831 +size 1444170 diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..0a560d50d88dfd4c7eac168e209ce3ecf5be502b --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc542bb6f251b5d41e0ca2a9c9bb7811e9ed3945d0d3e9e1fe3d72db8478a6a9 +size 1448685 diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..512a5a063bd83a0cff7b912bb015895bc6e69847 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log @@ -0,0 +1,276 @@ + +loaded agent with feature_type: 180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1 +[INFO] [1736299279.350383]: resetting environment +[INFO] [1736299279.355205]: cleared current plan +[INFO] [1736299279.356076]: moving to home +[INFO] [1736299280.358368]: reached home +[INFO] [1736299280.359669]: reset action history +[INFO] [1736299283.070417]: environment reset +Starting episode 0 +[INFO] [1736299283.077708]: resetting environment +[INFO] [1736299283.084698]: cleared current plan +[INFO] [1736299283.085590]: moving to home +[INFO] [1736299284.087847]: reached home +[INFO] [1736299284.089388]: reset action history +[INFO] [1736299286.799409]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +[INFO] [1736299289.269437]: resetting environment +[INFO] [1736299289.273691]: cleared current plan +[INFO] [1736299289.274461]: moving to home +[INFO] [1736299290.276973]: reached home +[INFO] [1736299290.278225]: reset action history +[INFO] [1736299292.987568]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/episode_rosbags/episode_0_2025-01-07-20-21-32.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2223909.log +started roslaunch server http://158.130.50.37:43927/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-1]: started with pid [2224194] +process[print_text-2]: started with pid [2224195] +process[rosbag_record-3]: started with pid [2224219] +started bagging! +preempting episode +[INFO] [1736299305.669183]: Storing episode... +[rosbag_record-3] killing on exit +[print_text-2] killing on exit +[pub_text-1] killing on exit +[INFO] [1736299306.391022]: Stored episode 1. +[INFO] [1736299306.391255]: Saving video... +[INFO] [1736299306.603232]: Video saved! +Starting episode 1 +[INFO] [1736299306.617027]: resetting environment +[INFO] [1736299306.626106]: cleared current plan +[INFO] [1736299306.626827]: moving to home +[INFO] [1736299309.729018]: reached home +[INFO] [1736299309.729450]: reset action history +[INFO] [1736299312.440328]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299320.537556]: resetting environment +[INFO] [1736299320.542964]: cleared current plan +[INFO] [1736299320.543160]: moving to home +[INFO] [1736299321.544579]: reached home +[INFO] [1736299321.544986]: reset action history +[INFO] [1736299324.253811]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/episode_rosbags/episode_1_2025-01-07-20-22-04.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2223909.log +started roslaunch server http://158.130.50.37:43359/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-4]: started with pid [2224340] +process[print_text-5]: started with pid [2224364] +process[rosbag_record-6]: started with pid [2224365] +started bagging! +For topic gripper_width: timestamp difference is -100451118 for nearest: 1736299338602728241 - target: 1736299338703179359 at idx 191 +For topic gripper_width: timestamp difference is 104160408 for nearest: 1736299344502699951 - target: 1736299344398539543 at idx 249 +[INFO] [1736299359.070266]: Storing episode... +[print_text-5] killing on exit +[rosbag_record-6] killing on exit +[pub_text-4] killing on exit +[INFO] [1736299359.845595]: Stored episode 2. +[INFO] [1736299359.846077]: Saving video... +[INFO] [1736299360.196462]: Video saved! +Starting episode 2 +[INFO] [1736299360.222950]: resetting environment +[INFO] [1736299360.239729]: cleared current plan +[INFO] [1736299360.239995]: moving to home +[INFO] [1736299363.944028]: reached home +[INFO] [1736299363.946973]: reset action history +[INFO] [1736299366.668830]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299373.745534]: resetting environment +[INFO] [1736299373.757316]: cleared current plan +[INFO] [1736299373.757564]: moving to home +[INFO] [1736299374.759383]: reached home +[INFO] [1736299374.759869]: reset action history +[INFO] [1736299377.467559]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/episode_rosbags/episode_2_2025-01-07-20-22-57.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2223909.log +started roslaunch server http://158.130.50.37:35487/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-7]: started with pid [2224487] +process[print_text-8]: started with pid [2224488] +process[rosbag_record-9]: started with pid [2224512] +started bagging! +For topic gripper_width: timestamp difference is 101112130 for nearest: 1736299382069337371 - target: 1736299381968225241 at idx 64 +[INFO] [1736299412.294830]: Storing episode... +[rosbag_record-9] killing on exit +[pub_text-7] killing on exit +[print_text-8] killing on exit +[INFO] [1736299413.013908]: Stored episode 3. +[INFO] [1736299413.017582]: Saving video... +[INFO] [1736299413.424240]: Video saved! +Starting episode 3 +[INFO] [1736299413.483134]: resetting environment +[INFO] [1736299413.510317]: cleared current plan +[INFO] [1736299413.511277]: moving to home +[INFO] [1736299417.220722]: reached home +[INFO] [1736299417.225490]: reset action history +[INFO] [1736299420.025344]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299427.896003]: resetting environment +[INFO] [1736299427.922461]: cleared current plan +[INFO] [1736299427.922670]: moving to home +[INFO] [1736299428.924712]: reached home +[INFO] [1736299428.925831]: reset action history +[INFO] [1736299431.710398]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/episode_rosbags/episode_3_2025-01-07-20-23-51.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2223909.log +started roslaunch server http://158.130.50.37:34767/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-10]: started with pid [2224652] +process[print_text-11]: started with pid [2224653] +process[rosbag_record-12]: started with pid [2224654] +started bagging! +For topic gripper_width: timestamp difference is -113815720 for nearest: 1736299446002669160 - target: 1736299446116484880 at idx 205 +For topic gripper_width: timestamp difference is -140634636 for nearest: 1736299461002671858 - target: 1736299461143306494 at idx 249 +[INFO] [1736299466.694301]: Storing episode... +[rosbag_record-12] killing on exit +[print_text-11] killing on exit +[pub_text-10] killing on exit +[INFO] [1736299467.438966]: Stored episode 4. +[INFO] [1736299467.444885]: Saving video... +[INFO] [1736299467.811127]: Video saved! +Starting episode 4 +[INFO] [1736299467.892358]: resetting environment +[INFO] [1736299467.911651]: cleared current plan +[INFO] [1736299467.915154]: moving to home +[INFO] [1736299471.617509]: reached home +[INFO] [1736299471.625669]: reset action history +[INFO] [1736299474.414543]: environment reset +Press Enter to continue... after reseting env. To rate prev episode, press 0 for failure and 1 for success +proceeding to start episode!Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +[INFO] [1736299482.340123]: resetting environment +[INFO] [1736299482.363129]: cleared current plan +[INFO] [1736299482.367720]: moving to home +[INFO] [1736299483.379678]: reached home +[INFO] [1736299483.382145]: reset action history +[INFO] [1736299486.171812]: environment reset +bag_path_name:=/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/episode_rosbags/episode_4_2025-01-07-20-24-46.bag +... logging to /home/leonmkim/.ros/log/a8a09c64-c64e-11ef-96d2-5defea869005/roslaunch-leonmkim-ROG-Strix-G15CS-G15CS-2223909.log +started roslaunch server http://158.130.50.37:34575/ +SUMMARY +======== +PARAMETERS + * /rosdistro: noetic + * /rosversion: 1.16.0 +NODES + / + print_text (rostopic/rostopic) + pub_text (rostopic/rostopic) + rosbag_record (rosbag/record) +ROS_MASTER_URI=http://localhost:11311 +process[pub_text-13]: started with pid [2224797] +process[print_text-14]: started with pid [2224798] +process[rosbag_record-15]: started with pid [2224799] +started bagging! +For topic gripper_width: timestamp difference is 104980613 for nearest: 1736299510669423486 - target: 1736299510564442873 at idx 249 +For topic gripper_width: timestamp difference is 106016033 for nearest: 1736299511969370478 - target: 1736299511863354445 at idx 249 +For topic gripper_width: timestamp difference is 107917023 for nearest: 1736299514302671624 - target: 1736299514194754601 at idx 249 +[INFO] [1736299521.354697]: Storing episode... +[pub_text-13] killing on exit +[print_text-14] killing on exit +[rosbag_record-15] killing on exit +[INFO] [1736299522.186357]: Stored episode 5. +[INFO] [1736299522.196667]: Saving video... +[INFO] [1736299522.739137]: Video saved! +[WARN] [1736299522.714845]: Plan exhausted +[WARN] [1736299522.751899]: Plan exhausted +[WARN] [1736299522.791854]: Plan exhausted +[INFO] [1736299522.782474]: resetting environment +[INFO] [1736299522.805856]: cleared current plan +[INFO] [1736299522.807144]: moving to home +[INFO] [1736299526.511818]: reached home +[INFO] [1736299526.515890]: reset action history +[INFO] [1736299529.317899]: environment reset +Evaluation finished. To wrap up, rate prev episode, press 0 for failure and 1 for success +proceeding to start episode! +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +Comuting lower confidence bound +Comuting upper confidence bound +proceeding to start episode! \ No newline at end of file diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/requirements.txt b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0012abb707c844944e9c10167241911549abc034 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/requirements.txt @@ -0,0 +1,340 @@ +Cython==3.0.10 +Farama-Notifications==0.0.4 +GitPython==3.1.43 +Jinja2==3.1.4 +Markdown==3.6 +MarkupSafe==2.1.5 +POT==0.7.0 +PyOpenGL==3.1.7 +PySocks==1.7.1 +PyYAML==6.0.1 +Pygments==2.18.0 +Rtree==1.3.0 +Werkzeug==3.0.3 +absl-py==2.1.0 +accelerate==0.33.0 +actionlib-msgs==1.13.0.post3 +actionlib==1.12.0 +actionlib==1.14.0 +aiohttp==3.9.5 +aiosignal==1.3.1 +angles==1.9.13 +antlr4-python3-runtime==4.9.3 +anyio==4.4.0 +asciitree==0.3.3 +async-timeout==4.0.3 +attrs==23.2.0 +autoprop==4.1.0 +beartype==0.18.5 +beautifulsoup4==4.12.3 +binomial_cis==0.0.11 +bondpy==1.8.6 +byol-pytorch==0.8.0 +cachetools==5.4.0 +camera-calibration-parsers==1.12.0 +camera-calibration==1.17.0 +cascadio==0.0.13 +catkin-pkg==1.0.0 +catkin==0.7.18 +catkin==0.8.10 +certifi==2024.7.4 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 +cloudpickle==3.0.0 +cmake==3.30.1 +colorlog==6.8.2 +contourpy==1.2.1 +controller-manager-msgs==0.20.0 +controller-manager==0.20.0 +coverage==7.6.0 +coveralls==4.0.1 +cv-bridge==1.16.2 +cycler==0.12.1 +datasets==2.20.0 +decorator==4.4.2 +deepdiff==7.0.1 +defusedxml==0.7.1 +diagnostic-analysis==1.11.0 +diagnostic-common-diagnostics==1.11.0 +diagnostic-updater==1.11.0 +diffusers==0.27.2 +dill==0.3.8 +distro==1.9.0 +dm-control==1.0.8 +dm-env==1.6 +dm-tree==0.1.8 +docker-pycreds==0.4.0 +docopt==0.6.2 +docutils==0.21.2 +dynamic-reconfigure==1.7.3 +einops==0.8.0 +embreex==2.17.7.post5 +empy==3.3.4 +etils==1.7.0 +exceptiongroup==1.2.2 +ezdxf==1.3.2 +fasteners==0.19 +filelock==3.15.4 +fonttools==4.53.1 +freetype-py==2.4.0 +frozenlist==1.4.1 +fsspec==2024.5.0 +gazebo_plugins==2.9.2 +gazebo_ros==2.9.2 +gdown==5.2.0 +gencpp==0.7.0 +geneus==3.0.0 +genlisp==0.4.18 +genmsg==0.5.12 +genmsg==0.6.0 +gennodejs==2.0.2 +genpy==0.6.14 +genpy==0.6.15 +geometry-msgs==1.13.0.post2 +gitdb==4.0.11 +glfw==2.7.0 +glooey==0.3.6 +gmsh==4.12.2 +gnupg==2.3.1 +google-auth-oauthlib==1.0.0 +google-auth==2.32.0 +grpcio==1.65.1 +gym-envs==0.0.1 +gym-notices==0.0.8 +gym==0.22.0 +gymnasium==0.29.1 +h11==0.14.0 +h5py==3.11.0 +hf_transfer==0.1.8 +httpcore==1.0.5 +httpx==0.27.0 +huggingface-hub==0.23.5 +hydra-core==1.3.2 +hydra-submitit-launcher==1.2.0 +idna==3.7 +image-geometry==1.16.2 +imageio-ffmpeg==0.5.1 +imageio==2.34.2 +importlib_metadata==8.2.0 +importlib_resources==6.4.0 +iniconfig==2.0.0 +interactive-markers==1.12.0 +joint-state-publisher-gui==1.15.1 +joint-state-publisher==1.15.1 +jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 +kiwisolver==1.4.5 +kornia==0.7.3 +kornia_rs==0.1.5 +labmaze==1.0.6 +laser_geometry==1.6.7 +lazy_loader==0.4 +lerobot==0.1.0 +lightning-utilities==0.11.6 +llvmlite==0.43.0 +lxml==5.2.2 +manifold3d==2.5.1 +mapbox-earcut==1.0.1 +markdown-it-py==3.0.0 +matplotlib==3.9.1 +mdurl==0.1.2 +meshio==5.3.5 +message-filters==1.16.0 +more-itertools==10.3.0 +moviepy==1.0.3 +mpmath==1.3.0 +mujoco==3.2.0 +multidict==6.0.5 +multiprocess==0.70.16 +natsort==8.4.0 +netifaces==0.11.0 +networkx==3.3 +nodeenv==1.9.1 +numba==0.60.0 +numcodecs==0.13.0 +numpy==1.26.4 +nvidia-cublas-cu12==12.1.3.1 +nvidia-cuda-cupti-cu12==12.1.105 +nvidia-cuda-nvrtc-cu12==12.1.105 +nvidia-cuda-runtime-cu12==12.1.105 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.0.2.54 +nvidia-curand-cu12==10.3.2.106 +nvidia-cusolver-cu12==11.4.5.107 +nvidia-cusparse-cu12==12.1.0.106 +nvidia-nccl-cu12==2.20.5 +nvidia-nvjitlink-cu12==12.5.82 +nvidia-nvtx-cu12==12.1.105 +oauthlib==3.2.2 +omegaconf==2.3.0 +openctm==0.0.5 +opencv-python==4.10.0.84 +ordered-set==4.1.0 +packaging==24.1 +pandas==2.2.2 +pillow==10.4.0 +pip==24.3.1 +platformdirs==4.2.2 +pluggy==1.5.0 +proglog==0.1.10 +protobuf==5.27.2 +psutil==6.0.0 +pyarrow-hotfix==0.6 +pyarrow==17.0.0 +pyasn1==0.6.0 +pyasn1_modules==0.4.0 +pyav==12.3.0 +pycollada==0.8 +pycparser==2.22 +pycryptodomex==3.21.0 +pyglet==1.5.29 +pyinstrument==4.6.2 +pymunk==6.8.1 +pyparsing==2.4.7 +pyrealsense2==2.54.2.5684 +pyribbit==0.1.46 +pyright==1.1.373 +pytest-beartype==0.0.2 +pytest-cov==5.0.0 +pytest==8.3.1 +python-dateutil==2.9.0.post0 +python-fcl==0.7.0.6 +python-qt-binding==0.4.4 +pytorch-lightning==2.4.0 +pytz==2024.1 +qt-dotgraph==0.4.2 +qt-gui-cpp==0.4.2 +qt-gui-py-common==0.4.2 +qt-gui==0.4.2 +referencing==0.35.1 +regex==2024.5.15 +requests-oauthlib==2.0.0 +requests==2.32.3 +rerun-sdk==0.17.0 +resource_retriever==1.12.7 +rich==13.7.1 +ros-numpy==0.0.5 +rosbag==1.16.0 +rosboost-cfg==1.15.8 +rosclean==1.15.8 +roscpp==1.15.11 +roscreate==1.15.8 +rosgraph-msgs==1.11.3.post2 +rosgraph==1.15.11 +rosgraph==1.16.0 +roslaunch==1.16.0 +roslib==1.14.7.post0 +roslib==1.15.8 +roslint==0.12.0 +roslz4==1.16.0 +rosmake==1.15.8 +rosmaster==1.16.0 +rosmsg==1.16.0 +rosnode==1.16.0 +rosparam==1.16.0 +rospkg==1.5.1 +rospy==1.15.11 +rospy==1.16.0 +rosservice==1.16.0 +rostest==1.16.0 +rostopic==1.16.0 +rosunit==1.15.8 +roswtf==1.16.0 +rpds-py==0.19.1 +rqt-console==0.4.12 +rqt-image-view==0.4.17 +rqt-logger-level==0.4.12 +rqt-moveit==0.5.11 +rqt-reconfigure==0.5.5 +rqt-robot-dashboard==0.5.8 +rqt-robot-monitor==0.5.15 +rqt-runtime-monitor==0.5.10 +rqt-rviz==0.7.0 +rqt-tf-tree==0.6.4 +rqt_action==0.4.9 +rqt_bag==0.5.1 +rqt_bag_plugins==0.5.1 +rqt_dep==0.4.12 +rqt_graph==0.4.14 +rqt_gui==0.5.3 +rqt_gui_py==0.5.3 +rqt_launch==0.4.9 +rqt_msg==0.4.10 +rqt_nav_view==0.5.7 +rqt_plot==0.4.13 +rqt_pose_view==0.5.11 +rqt_publisher==0.4.10 +rqt_py_common==0.5.3 +rqt_py_console==0.4.10 +rqt_robot_steering==0.5.12 +rqt_service_caller==0.4.10 +rqt_shell==0.4.11 +rqt_srv==0.4.9 +rqt_top==0.4.10 +rqt_topic==0.4.13 +rqt_web==0.4.10 +rsa==4.9 +ruff==0.5.4 +rviz==1.14.25 +safetensors==0.4.3 +scikit-image==0.24.0 +scikit-video==1.1.11 +scipy==1.14.0 +seaborn==0.13.2 +sensor-msgs==1.13.1 +sentry-sdk==2.11.0 +setproctitle==1.3.3 +setuptools==65.5.0 +shapely==2.0.5 +signature_dispatch==1.0.1 +six==1.16.0 +smach-ros==2.5.2 +smach==2.5.2 +smclib==1.8.6 +smmap==5.0.1 +sniffio==1.3.1 +soupsieve==2.5 +std-msgs==0.5.13.post0 +submitit==1.5.1 +svg.path==6.3 +sympy==1.13.1 +tensorboard-data-server==0.7.2 +tensorboard==2.14.0 +termcolor==2.4.0 +tf-conversions==1.13.2 +tf2-geometry-msgs==0.7.7 +tf2-kdl==0.7.7 +tf2-msgs==0.7.2.post3 +tf2-py==0.7.7 +tf2-ros==0.6.5 +tf2-ros==0.7.7 +tf2_py==0.6.5.post1 +tf==1.13.2 +tifffile==2024.7.24 +tomli==2.0.1 +topic-tools==1.16.0 +torch==2.4.0 +torchaug==0.5.2 +torchmetrics==1.4.0.post0 +torchvision==0.19.0 +tqdm==4.66.4 +trimesh==4.4.3 +triton==3.0.0 +typeguard==3.0.2 +typing_extensions==4.12.2 +tzdata==2024.1 +urchin==0.0.27 +urllib3==2.2.2 +vecrec==0.3.1 +vhacdx==0.0.8.post1 +wandb==0.17.5 +wheel==0.43.0 +xacro==1.14.18 +xatlas==0.0.9 +xxhash==3.4.1 +yarl==1.9.4 +zarr==2.18.2 +zipp==3.19.2 +zstandard==0.23.0 \ No newline at end of file diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-metadata.json b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..5b7693bad54773179ac31d847040bac07af9cf7e --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-metadata.json @@ -0,0 +1,92 @@ +{ + "os": "Linux-5.15.0-125-generic-x86_64-with-glibc2.31", + "python": "3.10.14", + "heartbeatAt": "2025-01-08T01:21:14.573756", + "startedAt": "2025-01-08T01:21:14.136932", + "docker": null, + "cuda": null, + "args": [ + "agent=diffusion", + "suite=frankagym", + "suite/frankagym_task@_global_=insertion" + ], + "state": "running", + "program": "/home/leonmkim/fish_leon/FISH/eval_robot.py", + "codePathLocal": null, + "codePath": "FISH/eval_robot.py", + "git": { + "remote": "https://github.com/leonmkim/fish_leon.git", + "commit": "29237eae507484d4aff7683bad40a564046255b9" + }, + "email": "leonmkim@seas.upenn.edu", + "root": "/home/leonmkim/fish_leon", + "host": "leonmkim-ROG-Strix-G15CS-G15CS", + "username": "leonmkim", + "executable": "/home/leonmkim/.pyenv/versions/lerobot/bin/python", + "cpu_count": 8, + "cpu_count_logical": 8, + "cpu_freq": { + "current": 3935.56425, + "min": 800.0, + "max": 4700.0 + }, + "cpu_freq_per_core": [ + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4494.112, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 4499.827, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + }, + { + "current": 3000.0, + "min": 800.0, + "max": 4700.0 + } + ], + "disk": { + "/": { + "total": 915.3232879638672, + "used": 572.616024017334 + } + }, + "gpu": "NVIDIA GeForce RTX 2070 SUPER", + "gpu_count": 1, + "gpu_devices": [ + { + "name": "NVIDIA GeForce RTX 2070 SUPER", + "memory_total": 8589934592 + } + ], + "memory": { + "total": 62.71595764160156 + } +} diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..6f1ef57f78466113803d410736e92f6e428ec686 --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json @@ -0,0 +1 @@ +{"eval/0_eval": {"_type": "video-file", "sha256": "bec6c78d6d5d8da97ef5b12f58d06b35eefbf3e99b7c219ecd57f38aa6f9592e", "size": 598726, "path": "media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4"}, "global_step": 579, "_timestamp": 1736299538.5800085, "_runtime": 264.43159556388855, "_step": 11, "eval/1_eval": {"_type": "video-file", "sha256": "407b26ce0b9569f5bbfddb38fc8f461c936cbaa4ff10b329dd8eafdd5f9b0b29", "size": 1442639, "path": "media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4"}, "eval/num_success": 4.0, "episode": 60.0, "eval/success_rate": 0.800000011920929, "eval/success_rate_lb_0.01": 0.3748520016670227, "eval/success_rate_ub_0.01": 0.9957208037376404, "eval/success_rate_lb_0.025": 0.3695220351219177, "eval/success_rate_ub_0.025": 0.9926368594169617, "eval/success_rate_lb_0.05": 0.44141441583633423, "eval/success_rate_ub_0.05": 0.9545822143554688, "eval/success_rate_lb_0.1": 0.4180309772491455, "eval/success_rate_ub_0.1": 0.9344762563705444, "eval/2_eval": {"_type": "video-file", "sha256": "a036985f900b09e578db306820019b257b10dfa7e10ec7292e161c4a7ddcf402", "size": 1446625, "path": "media/videos/eval/2_eval_2_a036985f900b09e578db.mp4"}, "eval/3_eval": {"_type": "video-file", "sha256": "c5460bdfba046140b658a20b3293bd3947a1f5cb63ddde4af2adb234ed176831", "size": 1444170, "path": "media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4"}, "eval/4_eval": {"_type": "video-file", "sha256": "dc542bb6f251b5d41e0ca2a9c9bb7811e9ed3945d0d3e9e1fe3d72db8478a6a9", "size": 1448685, "path": "media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4"}, "eval/success_rate_ci": {"_type": "table-file", "sha256": "8490adedbf62c026e30853c69820c3dd6a1c7da4ee481eab9e8dcf125e285d47", "size": 397, "artifact_path": "wandb-client-artifact://i9el3rdz6z7ehv8ljskaald2x226j0pp9rzopof3gimwkcosl6rjmtaruh7wjvzdi85rjk8esn4364f3hx1cvbrepdhzijf2gurajqxkccghkn5soh0hbfo8zwlomg45/eval/success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://9fhw5b90aywqrh8qrk78sf9owgkb77mrifthi9xydzmqqhtrvykrrlldifyskxvsafhlsnec4y8nu6up2tscwk7b649z6106c4vhew0wo48cskh6p7xkn7a8axdaiew3:latest/eval/success_rate_ci.table.json", "path": "media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json", "ncols": 9, "nrows": 1}, "total_num_successes": 41, "total_num_episodes": 60, "total_success_rate": 0.6833333333333333, "total_success_rate_lb_0.01": 0.5438502281816522, "total_success_rate_ub_0.01": 0.8056211470946375, "total_success_rate_lb_0.025": 0.5544648468385485, "total_success_rate_ub_0.025": 0.7868945374511087, "total_success_rate_lb_0.05": 0.5722976997350102, "total_success_rate_ub_0.05": 0.7759805619164684, "total_success_rate_lb_0.1": 0.6096675358493338, "total_success_rate_ub_0.1": 0.7589508182782594, "eval/total_success_rate_ci": {"_type": "table-file", "sha256": "579a029f256e694027d07331aab6927acd03bdfd688a5c4dd66989c6e1650c69", "size": 520, "artifact_path": "wandb-client-artifact://ztsa0sdpk8ajd9fu9c2zysuylplq85sfx5o804isyntsnh25wqlwuwlrj6hunv357bf9n1327tsk8kpt765l2ja3bp25rkfdes495hzfp8x2h0swluhfq3i1b4y5ygz7/eval/total_success_rate_ci.table.json", "_latest_artifact_path": "wandb-client-artifact://tr0vpsjvn4wkc1meuoctw397xjvbjp09xi6gjnowsedqlle2vr1f26w6y7pymwjn17sb2mfriqu820c2bivsxmckitkatm2z5kbc41teuoftivgmvo8wp5cy1h1kzw17:latest/eval/total_success_rate_ci.table.json", "path": "media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json", "ncols": 11, "nrows": 1}, "_wandb": {"runtime": 280}, "eval/total_success_rate": 0.6833333373069763, "eval/total_success_rate_lb_0.01": 0.5438502430915833, "eval/total_success_rate_ub_0.01": 0.8056211471557617, "eval/total_success_rate_lb_0.025": 0.5544648170471191, "eval/total_success_rate_ub_0.025": 0.7868945598602295, "eval/total_success_rate_lb_0.05": 0.5722976922988892, "eval/total_success_rate_ub_0.05": 0.775980532169342, "eval/total_success_rate_lb_0.1": 0.6096675395965576, "eval/total_success_rate_ub_0.1": 0.7589508295059204} \ No newline at end of file diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug-internal.log b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..98388bc48b22895665214d8f786742a6294e3d3f --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug-internal.log @@ -0,0 +1,789 @@ +2025-01-07 20:21:14,147 INFO StreamThr :2224087 [internal.py:wandb_internal():85] W&B internal server running at pid: 2224087, started at: 2025-01-07 20:21:14.146861 +2025-01-07 20:21:14,149 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status +2025-01-07 20:21:14,160 INFO WriterThread:2224087 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/run-w4b02c7s.wandb +2025-01-07 20:21:14,161 DEBUG SenderThread:2224087 [sender.py:send():379] send: header +2025-01-07 20:21:14,168 DEBUG SenderThread:2224087 [sender.py:send():379] send: run +2025-01-07 20:21:14,462 INFO SenderThread:2224087 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files +2025-01-07 20:21:14,462 INFO SenderThread:2224087 [sender.py:_start_run_threads():1188] run started: w4b02c7s with start time 1736299274.148413 +2025-01-07 20:21:14,479 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: check_version +2025-01-07 20:21:14,480 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: check_version +2025-01-07 20:21:14,555 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: run_start +2025-01-07 20:21:14,560 DEBUG HandlerThread:2224087 [system_info.py:__init__():26] System info init +2025-01-07 20:21:14,560 DEBUG HandlerThread:2224087 [system_info.py:__init__():41] System info init done +2025-01-07 20:21:14,560 INFO HandlerThread:2224087 [system_monitor.py:start():194] Starting system monitor +2025-01-07 20:21:14,561 INFO SystemMonitor:2224087 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-07 20:21:14,561 INFO HandlerThread:2224087 [system_monitor.py:probe():214] Collecting system info +2025-01-07 20:21:14,561 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started cpu monitoring +2025-01-07 20:21:14,562 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started disk monitoring +2025-01-07 20:21:14,563 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started gpu monitoring +2025-01-07 20:21:14,563 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started memory monitoring +2025-01-07 20:21:14,564 INFO SystemMonitor:2224087 [interfaces.py:start():188] Started network monitoring +2025-01-07 20:21:14,573 DEBUG HandlerThread:2224087 [system_info.py:probe():152] Probing system +2025-01-07 20:21:14,575 DEBUG HandlerThread:2224087 [system_info.py:_probe_git():137] Probing git +2025-01-07 20:21:14,581 DEBUG HandlerThread:2224087 [system_info.py:_probe_git():145] Probing git done +2025-01-07 20:21:14,581 DEBUG HandlerThread:2224087 [system_info.py:probe():200] Probing system done +2025-01-07 20:21:14,581 DEBUG HandlerThread:2224087 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-08T01:21:14.573756', 'startedAt': '2025-01-08T01:21:14.136932', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '29237eae507484d4aff7683bad40a564046255b9'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3935.56425, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4494.112, 'min': 800.0, 'max': 4700.0}, {'current': 4499.827, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 572.616024017334}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-07 20:21:14,581 INFO HandlerThread:2224087 [system_monitor.py:probe():224] Finished collecting system info +2025-01-07 20:21:14,581 INFO HandlerThread:2224087 [system_monitor.py:probe():227] Publishing system info +2025-01-07 20:21:14,581 DEBUG HandlerThread:2224087 [system_info.py:_save_code():44] Saving code +2025-01-07 20:21:14,584 DEBUG HandlerThread:2224087 [system_info.py:_save_code():67] Saving code done +2025-01-07 20:21:14,584 DEBUG HandlerThread:2224087 [system_info.py:_save_patches():84] Saving git patches +2025-01-07 20:21:14,650 DEBUG HandlerThread:2224087 [system_info.py:_save_patches():126] Saving git patches done +2025-01-07 20:21:14,651 INFO HandlerThread:2224087 [system_monitor.py:probe():229] Finished publishing system info +2025-01-07 20:21:14,654 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:21:14,654 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-07 20:21:14,654 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-07 20:21:14,654 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-07 20:21:14,876 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/1svrxmfh-wandb-metadata.json +2025-01-07 20:21:14,891 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: python_packages +2025-01-07 20:21:14,891 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: python_packages +2025-01-07 20:21:14,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:14,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:21:14,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:21:14,907 INFO wandb-upload_2:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/yhj813vc-diff.patch +2025-01-07 20:21:14,927 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/w5iryu7n-code/FISH/eval_robot.py +2025-01-07 20:21:14,933 INFO HandlerThread:2224087 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/tb" + save: true +} +control { + relay_id: "426b523202ad4154bb07363de3b5e08c" +} +_info { + stream_id: "w4b02c7s" +} + +2025-01-07 20:21:14,933 DEBUG HandlerThread:2224087 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-07 20:21:14,945 DEBUG SenderThread:2224087 [sender.py:send():379] send: telemetry +2025-01-07 20:21:14,998 DEBUG SenderThread:2224087 [sender.py:send():379] send: tbrecord +2025-01-07 20:21:14,999 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:21:14,999 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 with policy live +2025-01-07 20:21:15,173 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/wscof4s7-events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-metadata.json +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/requirements.txt +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code/FISH/eval_robot.py +2025-01-07 20:21:15,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/diff.patch +2025-01-07 20:21:15,465 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:15,465 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code +2025-01-07 20:21:15,465 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code/FISH +2025-01-07 20:21:15,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:16,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:17,463 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:17,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:18,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:19,350 DEBUG SenderThread:2224087 [sender.py:send():379] send: config +2025-01-07 20:21:19,350 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:19,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:20,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:21,464 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:21,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:22,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:23,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:24,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:24,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:25,465 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:25,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:26,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:27,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:28,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:29,467 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:29,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:21:29,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:21:29,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:30,036 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:30,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:31,467 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:31,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:32,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:33,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:34,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:35,468 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:35,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:35,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:36,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:37,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:38,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:39,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:40,893 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:40,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:41,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:42,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:43,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:44,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:21:44,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:21:44,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:45,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:46,395 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:46,615 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:21:46,616 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 with policy now +2025-01-07 20:21:46,616 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:21:46,617 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:21:46,617 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:21:46,619 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:21:46,851 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/brgutril-media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 +2025-01-07 20:21:46,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos +2025-01-07 20:21:47,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:21:47,743 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/aawlbvhc-media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 +2025-01-07 20:21:47,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:48,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:49,476 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:49,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:50,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:51,477 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:51,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:51,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:52,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:53,478 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:21:53,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:54,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:55,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:56,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:57,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:21:57,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:58,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:21:59,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:21:59,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:21:59,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:00,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:01,480 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:01,480 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:22:01,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:02,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:03,481 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:03,585 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:03,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:04,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:05,482 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:05,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:06,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:07,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:08,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:08,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:09,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:10,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:11,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:12,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:13,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:13,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:14,564 DEBUG SystemMonitor:2224087 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-07 20:22:14,564 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:22:14,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:22:14,894 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:22:14,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:15,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:16,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:17,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:18,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:19,841 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:19,900 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:20,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:21,487 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:21,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:22,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:23,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:24,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:24,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:25,489 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:25,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:26,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:27,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:28,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:29,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:22:29,894 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:22:29,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:30,045 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:30,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:31,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:32,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:33,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:34,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:35,894 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:35,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:36,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:37,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:38,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:39,903 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:40,245 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:22:40,248 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:22:40,248 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 with policy now +2025-01-07 20:22:40,248 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:22:40,248 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:22:40,249 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:22:40,494 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:22:40,494 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 +2025-01-07 20:22:40,495 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:22:40,520 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/i8mv5ivj-media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 +2025-01-07 20:22:40,806 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/nn13y66a-media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 +2025-01-07 20:22:40,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:41,249 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:41,495 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:41,904 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:42,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:43,906 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:44,565 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:22:44,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:22:44,901 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:22:44,945 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:45,496 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:45,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:46,669 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:46,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:47,904 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:48,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:49,498 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:49,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:50,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:51,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:51,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:52,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:53,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:54,338 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:22:54,338 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:22:54,339 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:22:54,499 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:22:54,499 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:22:54,739 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/0yikza4i-events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:22:54,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:55,500 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:55,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:56,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:22:56,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:57,500 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:57,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:58,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:22:59,501 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:22:59,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:22:59,896 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:22:59,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:00,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:01,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:02,003 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:02,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:03,503 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:03,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:04,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:05,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:06,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:07,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:07,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:08,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:09,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:10,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:11,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:12,895 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:12,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:13,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:14,565 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:23:14,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:23:14,896 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:23:14,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:15,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:16,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:17,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:18,003 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:18,910 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:19,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:20,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:21,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:22,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:23,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:23,911 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:24,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:25,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:26,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:27,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:28,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:28,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:29,896 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:23:29,897 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:23:29,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:30,912 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:31,913 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:32,921 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:33,458 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:23:33,458 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 with policy now +2025-01-07 20:23:33,471 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:23:33,472 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:23:33,472 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:23:33,476 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:23:33,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:33,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:23:33,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 +2025-01-07 20:23:33,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:23:33,709 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/v51k45zd-media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 +2025-01-07 20:23:33,914 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:33,990 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/js1noyat-media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 +2025-01-07 20:23:34,511 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:34,916 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:35,513 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:35,921 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:36,925 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:37,922 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:38,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:39,515 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:39,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:39,924 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:40,925 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:41,516 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:41,929 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:42,928 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:43,927 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:44,566 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:23:44,901 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:23:44,901 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:23:44,945 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:44,982 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:45,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:46,941 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:47,931 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:48,182 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:23:48,182 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:23:48,184 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:23:48,518 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:23:48,518 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:23:48,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:49,518 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:49,944 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:50,905 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:50,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:51,519 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:51,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:52,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:53,520 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:23:53,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:54,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:55,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:56,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:23:56,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:57,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:58,933 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:23:59,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:23:59,897 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:23:59,941 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:00,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:01,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:02,018 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:02,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:03,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:04,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:05,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:06,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:07,245 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:07,524 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:07,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:08,934 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:09,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:10,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:11,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:12,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:12,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:13,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:14,566 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:24:14,897 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:24:14,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:24:14,941 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:15,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:16,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:17,935 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:17,958 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:18,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:19,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:20,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:21,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:22,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:23,302 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:23,530 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:23,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:24,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:25,936 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:26,937 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:27,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:27,880 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:24:27,880 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 with policy now +2025-01-07 20:24:27,888 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:24:27,888 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:24:27,888 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:24:27,895 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:24:27,942 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:28,173 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/xg728rec-media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 +2025-01-07 20:24:28,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:24:28,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 +2025-01-07 20:24:28,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:24:28,833 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/20hvtfdl-media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 +2025-01-07 20:24:28,915 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:28,944 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:29,532 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:29,899 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:24:29,899 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:24:29,943 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:30,957 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:31,954 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:32,951 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:33,534 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:33,953 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:34,418 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:34,958 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:35,534 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:35,958 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:36,953 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:37,960 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:38,964 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:39,902 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:39,967 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:40,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:41,963 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:42,537 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:24:42,815 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/zr5gfofa-events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:24:42,971 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:43,247 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:24:43,247 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:24:43,247 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:24:43,537 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:43,537 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:24:43,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:44,567 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:24:44,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:24:44,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:24:44,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:45,043 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:45,538 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:45,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:46,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:47,539 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:47,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:48,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:49,540 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:24:49,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:50,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:50,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:51,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:52,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:53,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:54,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:55,965 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:56,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:24:56,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:57,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:58,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:24:59,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:24:59,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:24:59,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:00,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:01,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:01,971 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:02,966 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:03,967 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:04,967 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:05,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:06,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:07,907 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:07,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:08,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:09,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:10,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:11,547 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:11,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:12,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:13,017 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:13,548 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:13,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:14,567 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:25:14,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:25:14,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:25:14,968 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:15,549 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:15,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:16,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:17,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:18,050 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:18,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:19,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:20,969 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:21,974 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:22,780 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:25:22,780 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 with policy now +2025-01-07 20:25:22,781 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:25:22,782 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:25:22,782 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:22,801 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:22,973 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:23,069 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/28l1v0g1-media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 +2025-01-07 20:25:23,551 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:23,551 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:25:23,552 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 +2025-01-07 20:25:23,552 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval +2025-01-07 20:25:23,807 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:23,880 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/q1iq46wv-media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 +2025-01-07 20:25:23,976 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:24,981 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:25,552 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:25,982 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:27,001 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:27,553 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:27,987 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:28,980 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:29,321 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:29,908 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:25:29,908 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:25:29,979 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:30,982 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:31,554 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:31,984 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:32,992 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:33,992 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:34,922 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:34,995 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:36,002 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:37,002 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:37,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:38,072 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:25:38,072 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:25:38,230 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:25:38,522 INFO wandb-upload_1:2224087 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmprnm7pcgi +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval +2025-01-07 20:25:38,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table +2025-01-07 20:25:38,663 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-07 20:25:38,676 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: partial_history +2025-01-07 20:25:38,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:39,075 INFO SenderThread:2224087 [sender.py:send_request_log_artifact():1525] logged artifact run-w4b02c7s-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM4Mzc5MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzMTIwMg==', 'latestArtifact': None}} +2025-01-07 20:25:39,075 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:25:39,075 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json with policy now +2025-01-07 20:25:39,075 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:25:39,075 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,076 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,076 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:25:39,076 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,080 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,080 DEBUG SenderThread:2224087 [sender.py:send():379] send: telemetry +2025-01-07 20:25:39,080 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,081 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,081 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,081 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,081 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,082 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,082 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,082 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,082 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,083 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,083 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,083 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,083 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,083 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,084 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,084 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,084 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,084 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,085 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,085 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,085 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,085 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,085 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: log_artifact +2025-01-07 20:25:39,251 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/tuf9qeia-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-07 20:25:39,481 INFO wandb-upload_0:2224087 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpqcd0wrrf +2025-01-07 20:25:39,503 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/d6bqe7eu-media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-07 20:25:39,557 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:39,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:25:39,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:25:39,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json +2025-01-07 20:25:39,559 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval +2025-01-07 20:25:39,863 INFO SenderThread:2224087 [sender.py:send_request_log_artifact():1525] logged artifact run-w4b02c7s-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQyNjM4MzgxNg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUzMTgzMTIwMw==', 'latestArtifact': None}} +2025-01-07 20:25:39,863 DEBUG SenderThread:2224087 [sender.py:send():379] send: files +2025-01-07 20:25:39,863 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json with policy now +2025-01-07 20:25:39,865 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:25:39,865 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:25:39,866 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:39,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:40,049 INFO wandb-upload_2:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/8mliszsk-media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json +2025-01-07 20:25:40,306 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/9ys5wxb3-media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json +2025-01-07 20:25:40,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:25:40,916 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:40,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:41,558 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:25:41,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:42,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:43,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:44,568 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:25:44,898 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: stop_status +2025-01-07 20:25:44,898 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: stop_status +2025-01-07 20:25:44,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:45,978 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:45,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:46,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:47,997 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:48,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:49,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:50,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:51,916 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:25:51,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:52,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:53,998 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:25:54,889 DEBUG SenderThread:2224087 [sender.py:send():379] send: exit +2025-01-07 20:25:54,889 INFO SenderThread:2224087 [sender.py:send_exit():586] handling exit code: 0 +2025-01-07 20:25:54,889 INFO SenderThread:2224087 [sender.py:send_exit():588] handling runtime: 280 +2025-01-07 20:25:54,890 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:25:54,890 INFO SenderThread:2224087 [sender.py:send_exit():594] send defer +2025-01-07 20:25:54,890 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,890 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-07 20:25:54,890 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:25:54,890 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-07 20:25:54,890 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 1 +2025-01-07 20:25:54,890 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,890 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-07 20:25:54,891 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:25:54,891 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-07 20:25:54,891 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 2 +2025-01-07 20:25:54,891 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,891 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-07 20:25:54,891 INFO HandlerThread:2224087 [system_monitor.py:finish():203] Stopping system monitor +2025-01-07 20:25:54,891 DEBUG SystemMonitor:2224087 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-07 20:25:54,891 DEBUG SystemMonitor:2224087 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-07 20:25:54,892 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined cpu monitor +2025-01-07 20:25:54,892 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined disk monitor +2025-01-07 20:25:54,945 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined gpu monitor +2025-01-07 20:25:54,945 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined memory monitor +2025-01-07 20:25:54,945 INFO HandlerThread:2224087 [interfaces.py:finish():200] Joined network monitor +2025-01-07 20:25:54,946 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:25:54,946 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-07 20:25:54,946 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 3 +2025-01-07 20:25:54,946 DEBUG SenderThread:2224087 [sender.py:send():379] send: stats +2025-01-07 20:25:54,946 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,946 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-07 20:25:54,946 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:25:54,946 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-07 20:25:54,946 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 4 +2025-01-07 20:25:54,947 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:25:54,947 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-07 20:25:55,565 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:25:57,566 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:26:01,320 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:01,321 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:01,321 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: keepalive +2025-01-07 20:26:01,321 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:01,322 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-07 20:26:01,322 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 5 +2025-01-07 20:26:01,322 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:01,322 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:01,322 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-07 20:26:01,323 DEBUG SenderThread:2224087 [sender.py:send():379] send: history +2025-01-07 20:26:01,323 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: summary_record +2025-01-07 20:26:01,323 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:26:01,324 DEBUG SenderThread:2224087 [sender.py:send():379] send: summary +2025-01-07 20:26:01,327 INFO SenderThread:2224087 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-07 20:26:01,328 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:01,328 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-07 20:26:01,328 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 6 +2025-01-07 20:26:01,328 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:01,328 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-07 20:26:01,328 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:01,328 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-07 20:26:01,331 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: status_report +2025-01-07 20:26:01,445 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 7 +2025-01-07 20:26:01,445 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:01,445 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-07 20:26:01,445 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:01,446 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-07 20:26:01,567 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml +2025-01-07 20:26:01,567 INFO Thread-12 :2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:26:01,890 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:04,759 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 8 +2025-01-07 20:26:04,759 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:04,760 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:04,760 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-07 20:26:04,760 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:04,760 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-07 20:26:04,760 INFO SenderThread:2224087 [job_builder.py:build():440] Attempting to build job artifact +2025-01-07 20:26:04,762 INFO SenderThread:2224087 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-07 20:26:04,764 INFO SenderThread:2224087 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-07 20:26:04,779 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 9 +2025-01-07 20:26:04,779 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:04,779 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-07 20:26:04,779 DEBUG SenderThread:2224087 [sender.py:send():379] send: artifact +2025-01-07 20:26:04,891 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:04,993 INFO SenderThread:2224087 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQyNTc5MjM4Ng==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQyNjE0MDMwNQ==', 'versionIndex': 437}}} +2025-01-07 20:26:04,993 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:04,993 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-07 20:26:04,993 INFO SenderThread:2224087 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-07 20:26:05,568 INFO SenderThread:2224087 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml config.yaml +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json wandb-summary.json +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/requirements.txt requirements.txt +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-metadata.json wandb-metadata.json +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log output.log +2025-01-07 20:26:05,569 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/diff.patch diff.patch +2025-01-07 20:26:05,570 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 media/videos/eval/4_eval_4_dc542bb6f251b5d41e0c.mp4 +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 media/videos/eval/1_eval_1_407b26ce0b9569f5bbfd.mp4 +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 media/videos/eval/0_eval_0_bec6c78d6d5d8da97ef5.mp4 +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 media/videos/eval/3_eval_3_c5460bdfba046140b658.mp4 +2025-01-07 20:26:05,573 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 media/videos/eval/2_eval_2_a036985f900b09e578db.mp4 +2025-01-07 20:26:05,575 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json media/table/eval/total_success_rate_ci_6_579a029f256e694027d0.table.json +2025-01-07 20:26:05,575 INFO SenderThread:2224087 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json media/table/eval/success_rate_ci_5_8490adedbf62c026e308.table.json +2025-01-07 20:26:05,575 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 10 +2025-01-07 20:26:05,575 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:05,576 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:05,578 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-07 20:26:05,578 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:05,578 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-07 20:26:05,578 INFO SenderThread:2224087 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:26:05,753 INFO wandb-upload_2:2224087 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/config.yaml +2025-01-07 20:26:05,808 INFO wandb-upload_3:2224087 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/requirements.txt +2025-01-07 20:26:05,808 INFO wandb-upload_0:2224087 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/wandb-summary.json +2025-01-07 20:26:05,814 INFO wandb-upload_1:2224087 [upload_job.py:push():130] Uploaded file /tmp/tmp28suo57nwandb/dnrqfq78-events.out.tfevents.1736299274.leonmkim-ROG-Strix-G15CS-G15CS.2223909.0 +2025-01-07 20:26:05,820 INFO wandb-upload_4:2224087 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/files/output.log +2025-01-07 20:26:05,892 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:05,892 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:06,021 INFO Thread-11 (_thread_body):2224087 [sender.py:transition_state():614] send defer: 11 +2025-01-07 20:26:06,021 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:06,021 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-07 20:26:06,021 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:06,021 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-07 20:26:06,021 INFO SenderThread:2224087 [file_pusher.py:join():175] waiting for file pusher +2025-01-07 20:26:06,023 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 12 +2025-01-07 20:26:06,023 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:06,023 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-07 20:26:06,023 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:06,023 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-07 20:26:06,023 INFO SenderThread:2224087 [file_stream.py:finish():601] file stream finish called +2025-01-07 20:26:06,091 INFO SenderThread:2224087 [file_stream.py:finish():605] file stream finish is done +2025-01-07 20:26:06,091 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 13 +2025-01-07 20:26:06,091 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:06,091 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-07 20:26:06,091 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:06,091 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-07 20:26:06,091 INFO SenderThread:2224087 [sender.py:transition_state():614] send defer: 14 +2025-01-07 20:26:06,091 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: defer +2025-01-07 20:26:06,092 INFO HandlerThread:2224087 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-07 20:26:06,092 DEBUG SenderThread:2224087 [sender.py:send():379] send: final +2025-01-07 20:26:06,092 DEBUG SenderThread:2224087 [sender.py:send():379] send: footer +2025-01-07 20:26:06,092 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: defer +2025-01-07 20:26:06,092 INFO SenderThread:2224087 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-07 20:26:06,092 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:06,093 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:06,093 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: server_info +2025-01-07 20:26:06,093 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-07 20:26:06,093 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: get_summary +2025-01-07 20:26:06,094 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-07 20:26:06,094 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: server_info +2025-01-07 20:26:06,096 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-07 20:26:06,144 DEBUG SenderThread:2224087 [sender.py:send_request():406] send_request: poll_exit +2025-01-07 20:26:06,144 INFO MainThread:2224087 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-07 20:26:06,144 INFO MainThread:2224087 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-07 20:26:06,145 INFO MainThread:2224087 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-07 20:26:06,145 DEBUG HandlerThread:2224087 [handler.py:handle_request():158] handle_request: shutdown +2025-01-07 20:26:06,145 INFO HandlerThread:2224087 [handler.py:finish():882] shutting down handler +2025-01-07 20:26:07,094 INFO WriterThread:2224087 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/run-w4b02c7s.wandb +2025-01-07 20:26:07,144 INFO SenderThread:2224087 [sender.py:finish():1615] shutting down sender +2025-01-07 20:26:07,144 INFO SenderThread:2224087 [file_pusher.py:finish():169] shutting down file pusher +2025-01-07 20:26:07,144 INFO SenderThread:2224087 [file_pusher.py:join():175] waiting for file pusher diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug.log b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..6795a233759d3fbc5adb02cc4d2bcf59e06be7af --- /dev/null +++ b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug.log @@ -0,0 +1,31 @@ +2025-01-07 20:21:14,138 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Current SDK version is 0.17.5 +2025-01-07 20:21:14,138 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Configure stats pid to 2223909 +2025-01-07 20:21:14,138 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/.config/wandb/settings +2025-01-07 20:21:14,138 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Loading settings from /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/settings +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'FISH/eval_robot.py', 'program_abspath': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py'} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_setup.py:_flush():76] Applying login settings: {} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:_log_setup():529] Logging user logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug.log +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:_log_setup():530] Logging internal logs to /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/logs/debug-internal.log +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:init():569] calling init triggers +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:init():576] wandb.init called with sweep_config: {} +config: {'root_dir': '/home/leonmkim/fish_leon', 'replay_buffer_size': 150000, 'replay_buffer_num_workers': 2, 'nstep': 3, 'batch_size': 128, 'seed': 1, 'dataset_shuffle_seed': 0, 'device': 'cuda', 'save_video': True, 'save_train_video': True, 'use_tb': True, 'use_wandb': True, 'wandb_run_id': '1013_1', 'wandb_notes': '1013_1_restarted_11', 'eval': True, 'true_action_history': False, 'train_pad_after': 4, 'process_contact_features': True, 'obs_type': 'pixels', 'use_color': True, 'use_depth': True, 'use_masks': True, 'mask_list': ['EE_obj_mask'], 'mask_representation': 'channels', 'crop_hw': [144, 144], 'crop_down_offset': 48, 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'add_crop_binary_mask': False, 'add_coord_conv_map': False, 'use_context_color': False, 'use_context_depth': False, 'use_context_segmask': False, 'context_color_crop_type': None, 'context_depth_crop_type': None, 'context_segmask_crop_type': None, 'context_add_crop_binary_mask': False, 'context_add_coord_conv_map': False, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'max_contact_prob': 0.1, 'max_depth': 2.0, 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'dtc_adaptive_normalization': False, 'mask_normals_within_sdf': True, 'adaptive_normals_mask': True, 'learnable_contact_preprocess_params': True, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'encoder_type': 'small', 'debug_timestamps': False, 'open_loop': False, 'action_trajectories': True, 'stop_after_action': False, 'interpolation_frequency': 25, 'policy_frequency': 5, 'wait_for_new_camera_frames': True, 'baseline': False, 'train_demo_idxs_list_or_num': -1, 'log_train_every_steps': 25, 'name_of_expert_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'expert_dataset_dirpath': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'store_dataset_in_memory': False, 'expert_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'action_key': 'action_trajectory_25hz', 'semantic_demo_grouping_name': 'semantic_demo_grouping.yaml', 'semantic_demo_grouping': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/semantic_demo_grouping.yaml', 'include_groups_list': 'all', 'expert_dataset_config': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demo_config.yaml', 'name_of_valid_demo': '120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act', 'valid_dataset_dir': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'valid_demo_idxs_list_or_num': None, 'val_num_groups': 5, 'load_bc': True, 'checkpoint_epoch_list': [99, 199, 299, 399, 499, 599, 699, 799, 899, 999, 1249, 1499, 1749, 1999, 2999, 3999, 4999, 5999, 6999, 7999, 8999, 9999], 'snapshot_root_dir': '/mnt/grasp_high_usage/leonmkim/contact_estimation/FISH', 'save_snapshot': True, 'save_last_snapshot': True, 'save_snapshot_when_done': True, 'top_k_checkpoints': 5, 'save_snapshot_link_to_weights_dir': 'deprecated', 'bc_regularize': False, 'bc_weight_type': 'qfilter', 'experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'agent': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgent', 'name': 'diffusion_policy', 'load_checkpoint': True, 'device': 'cuda', 'n_obs_steps': 1, 'suite_name': 'frankagym', 'obs_type': 'pixels', 'enable_arm': True, 'enable_camera': True, 'use_tb': True, 'desired_image_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'config': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}}, 'suite': {'suite': 'frankagym', 'name': 'frankagym', 'frame_stack': 1, 'action_repeat': 1, 'discount': 0.99, 'hidden_dim': 1024, 'num_train_frames': 2010, 'num_seed_frames': 260, 'num_train_epochs': 5000, 'validate_every_epochs': 100, 'validate_diffusion_on_action_loss_every_epochs': 500, 'train_eval_diffusion_on_action_loss_every_epochs': 500, 'check_topk_every_epochs': 10, 'save_snapshot_every_epochs': 5000, 'eval_every_frames': 2000, 'num_eval_episodes': 5, 'save_snapshot': True, 'wait_for_user_to_start_episode': True, 'task_make_fn': {'_target_': 'suite.frankagym.make', 'name': 'FrankaInsertion-v1', 'height': 240, 'width': 320, 'frame_stack': 1, 'action_repeat': 1, 'seed': 1, 'enable_arm': True, 'enable_gripper': True, 'start_with_gripper_open': True, 'enable_camera': True, 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'contact_estimation_model_ckpt_path': '~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt', 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'device': 'cuda', 'interpolation_frequency': 25, 'policy_frequency': 5, 'debug_timestamps': False, 'stop_after_action': False, 'open_loop': False, 'wait_for_new_camera_frames': True, 'action_key': 'action_trajectory_25hz', 'action_trajectory_horizon': 36, 'action_trajectories': True, 'path_to_zarr_dataset': '/home/leonmkim/fish_leon/FISH/expert_demos/frankagym/FrankaInsertion-v1/120_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act/demos.zarr', 'agent_policy_cfg': {'_target_': 'agent.diffusion_policy.DiffusionPolicyAgentConfig', 'compile': False, 'device': 'cuda', 'cam_resize_shape': [13, 180, 240], 'orig_cam_shape': [3, 240, 320], 'policy_frequency': 5, 'interpolation_frequency': 25, 'policy_cfg': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig', 'n_obs_steps': 1, 'horizon': 36, 'n_action_steps': 36, 'output_shapes': {'action': [7]}, 'input_normalization_modes': {'observation.image': 'mean_std', 'observation.state': 'min_max', 'observation.action_history': 'min_max'}, 'output_normalization_modes': {'action': 'min_max'}, 'vision_backbone': 'resnet18', 'pretrained_backbone_weights': None, 'transforms': [{'_target_': 'torchaug.transforms.RandomAffine', 'degrees': [-5, 5], 'translate': [0.05, 0.05], 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}, {'_target_': 'torchaug.transforms.RandomColorJitter', 'brightness': 0.3, 'contrast': 0.4, 'saturation': 0.5, 'hue': 0.08, 'batch_transform': True, 'num_chunks': -1, 'batch_inplace': True}], 'use_group_norm': True, 'spatial_softmax_num_keypoints': 32, 'action_history_encoder_config': {'_target_': 'lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig', 'in_channels': 7, 'out_channels': 32, 'history_length': 6, 'kernel_size': 5, 'downsample_kernel_size': 3, 'downsample_stride': 2, 'downsample_padding': 1}, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'n_groups': 8, 'diffusion_step_embed_dim': 128, 'use_film_scale_modulation': True, 'noise_scheduler_type': 'DDIM', 'beta_schedule': 'squaredcos_cap_v2', 'beta_start': 0.0001, 'beta_end': 0.02, 'prediction_type': 'epsilon', 'clip_sample': True, 'clip_sample_range': 1.0, 'num_train_timesteps': 50, 'num_inference_steps': 10, 'do_mask_loss_for_padding': False, 'input_shapes': {'observation.image': [13, 180, 240], 'context_observation.image': [13, 180, 240], 'observation.state': [8], 'observation.action_history': [7]}}, 'train_cfg': {'_target_': 'utils.TrainConfig', 'lr': 0.0001, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'adam_betas': [0.95, 0.999], 'adam_eps': 1e-08, 'adam_weight_decay': 1e-06, 'grad_clip_norm': 10, 'offline_steps': 1000000, 'use_amp': True}, 'observation_cfg': {'_target_': 'agent.encoder.VisualFeatureSet', 'use_depth': True, 'use_color': True, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': True, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}, 'context_input_config': {'_target_': 'agent.encoder.ContextInputConfig', 'use_color': False, 'use_depth': False, 'mask_input_dict': {'_target_': 'agent.encoder.MaskInputDict', 'enable': False, 'representation': 'channels', 'mask_list': ['EE_obj_mask']}, 'crop_input_config': {'_target_': 'agent.encoder.CropInputConfig', 'color_crop_type': None, 'depth_crop_type': None, 'segmask_crop_type': None, 'crop_hw': [144, 144], 'crop_down_offset': 48, 'add_crop_binary_mask': False, 'add_coord_conv_map': False}}, 'mask_soft_approx_scheduler_config': {'_target_': 'agent.encoder.MaskSoftApproxSchedulerConfig', 'num_steps': 40000, 'initial_value': 10.0, 'final_value': 1000.0, 'interpolation_scheme': 'cosine'}, 'use_contact_map': True, 'use_sdf_maps': True, 'use_normals_maps': True, 'which_objects': 'both', 'grasped_dtc_max_value': 0.2, 'env_dtc_max_value': 0.4, 'grasped_normals_mask_max_dtc_value': 0.2, 'env_normals_mask_max_dtc_value': 0.4, 'clamp_dtc': True, 'max_contact_prob': 0.1, 'mask_normals_within_sdf': True, 'dtc_adaptive_normalization': False, 'adaptive_normals_mask': True, 'max_depth': 2.0, 'image_shape': [13, 180, 240], 'learnable_contact_preprocess_params': True, 'learning_rate': 0.0001, 'weight_decay': 0.0, 'contact_model_name': 'local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9', 'zero_centered': False}}, 'true_action_history': False}}, 'num_train_frames_bc': 50000, 'num_train_frames_drq': 1100000, 'stddev_schedule_drq': 'linear(1.0,0.1,100000)', 'task_name': 'FrankaInsertion-v1', 'num_train_frames_vinn': 25000, 'num_train_frames_diffusion': 1000000, 'num_train_epochs_bc': 5000, 'num_train_epochs_diffusion': 15000, 'validate_every_epochs_bc': 5, 'validate_every_epochs_diffusion': 250, 'validate_diffusion_on_action_loss_every_epochs': 250, 'train_eval_diffusion_on_action_loss_every_epochs': 250, 'check_topk_every_epochs': 5, 'check_topk_every_epochs_diffusion': 250, 'save_snapshot_every_epochs_diffusion': 1500, 'x_limit': [0.2, 0.7], 'y_limit': [-0.4, 0.4], 'z_limit': [-0.05, 0.55], 'home_displacement': [0.55, 0.0, 0.55, 180.0, 0.0, 0.0], 'enable_gripper': True, 'start_with_gripper_open': True, 'offset_mask': [1, 1, 1, 1, 1, 1], 'path_to_depth_extrinsics': '~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy', 'feature_type': '180x240_1_RGB_D_2.0_msk_channels_EE_obj_mask_cntct_0.1_DTC_clmpd_lrnbl_nrmls_DTCmask_adpt_lrnbl_both_lr_0.0001_wd_0.0_local_multitask_outhd64all_home_crop_h144w144d48_mask_ctxtmask_seed_220979_epoch_9_acthst_hst6_out32_dwnkrnl3_dwnstrd2_dwnpd1', 'save_buffer': True, 'num_eval': 5, 'random_start': False, 'eval_starts': '/home/leonmkim/fish_leon/FISH/eval_starts/frankagym_pixels/FrankaInsertion-v1', 'num_valid_demos': None, 'load_checkpoint': True, 'checkpoint_epoch': 12000, 'load_residual_weight': False, 'checkpoint_root_dir': '/home/leonmkim/fish_leon/FISH', 'checkpoint_weight_dir': '/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1', 'residual_weight': '/home/leonmkim/fish_leon/FISH/weights/frankagym_pixels/FrankaInsertion-v1/weight.pt', 'final_experiment_dir': './exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s'} +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:init():619] starting backend +2025-01-07 20:21:14,139 INFO MainThread:2223909 [wandb_init.py:init():623] setting up manager +2025-01-07 20:21:14,146 INFO MainThread:2223909 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2025-01-07 20:21:14,147 INFO MainThread:2223909 [wandb_init.py:init():631] backend started and connected +2025-01-07 20:21:14,159 INFO MainThread:2223909 [wandb_init.py:init():720] updated telemetry +2025-01-07 20:21:14,166 INFO MainThread:2223909 [wandb_init.py:init():753] communicating run to backend with 90.0 second timeout +2025-01-07 20:21:14,479 INFO MainThread:2223909 [wandb_run.py:_on_init():2435] communicating current version +2025-01-07 20:21:14,549 INFO MainThread:2223909 [wandb_run.py:_on_init():2444] got version response upgrade_message: "wandb version 0.19.2 is available! To upgrade, please run:\n $ pip install wandb --upgrade" + +2025-01-07 20:21:14,550 INFO MainThread:2223909 [wandb_init.py:init():804] starting run threads in backend +2025-01-07 20:21:14,892 INFO MainThread:2223909 [wandb_run.py:_console_start():2413] atexit reg +2025-01-07 20:21:14,892 INFO MainThread:2223909 [wandb_run.py:_redirect():2255] redirect: wrap_raw +2025-01-07 20:21:14,892 INFO MainThread:2223909 [wandb_run.py:_redirect():2320] Wrapping output streams. +2025-01-07 20:21:14,892 INFO MainThread:2223909 [wandb_run.py:_redirect():2345] Redirects installed. +2025-01-07 20:21:14,893 INFO MainThread:2223909 [wandb_init.py:init():847] run started, returning control to user process +2025-01-07 20:21:14,894 INFO MainThread:2223909 [wandb_run.py:_tensorboard_callback():1544] tensorboard callback: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/w4b02c7s/tb, True +2025-01-07 20:21:19,349 INFO MainThread:2223909 [wandb_run.py:_config_callback():1382] config_cb None None {'grasped_obj_name': 'modelsys', 'left_book_slot': 'twodim'} +2025-01-07 20:26:07,146 WARNING MsgRouterThr:2223909 [router.py:message_loop():77] message_loop has been closed diff --git a/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/run-w4b02c7s.wandb b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/run-w4b02c7s.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6404097256d8cc49ebfc6c5cab63e4f364f2e676 Binary files /dev/null and b/w4b02c7s/wandb/run-20250107_202114-w4b02c7s/run-w4b02c7s.wandb differ diff --git a/x3d67guz/.hydra/config.yaml b/x3d67guz/.hydra/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e7a10df419e9a14bfc8d2bca5db66c7f85b18e3 --- /dev/null +++ b/x3d67guz/.hydra/config.yaml @@ -0,0 +1,350 @@ +root_dir: /home/${oc.env:USER}/fish_leon +nstep: 3 +seed: 41 +dataset_shuffle_seed: ${seed} +device: cuda +save_video: true +save_buffer: true +use_tb: true +baseline: false +use_wandb: true +eval: true +process_contact_features: ${eval} +obs_type: pixels +use_color: true +use_depth: true +use_masks: false +mask_list: +- EE_obj_mask +mask_representation: channels +crop_hw: +- 144 +- 144 +crop_down_offset: 48 +color_crop_type: null +depth_crop_type: null +segmask_crop_type: null +add_crop_binary_mask: false +add_coord_conv_map: false +use_context_color: false +use_context_depth: false +use_context_segmask: false +context_color_crop_type: null +context_depth_crop_type: null +context_segmask_crop_type: null +context_add_crop_binary_mask: false +context_add_coord_conv_map: false +use_contact_map: false +use_sdf_maps: false +use_normals_maps: false +which_objects: both +max_contact_prob: 0.1 +max_depth: 2.0 +grasped_dtc_max_value: 0.105 +env_dtc_max_value: 0.425 +grasped_normals_mask_max_dtc_value: 0.105 +env_normals_mask_max_dtc_value: 0.425 +clamp_dtc: true +dtc_adaptive_normalization: false +mask_normals_within_sdf: true +adaptive_normals_mask: true +learnable_contact_preprocess_params: false +contact_model_name: local_multitask_outhd64all_home_crop_h144w144d48_ctxt_seed_183386_epoch_9 +contact_estimation_model_ckpt_path: ~/fish_leon/contact_estimation/artifacts/175604_2/checkpoints/epoch=09-val_loss=0.00.ckpt +num_eval: 5 +debug_timestamps: false +open_loop: false +action_trajectories: true +stop_after_action: false +interpolation_frequency: 25 +policy_frequency: 5 +wait_for_new_camera_frames: true +random_start: false +eval_starts: ${root_dir}/FISH/eval_starts/${suite.name}_${obs_type}/${task_name} +train_demo_idxs_list_or_num: null +num_valid_demos: null +val_num_groups: 3 +name_of_expert_demo: 112_240x320_all_twodim_left_to_right_annotated_start_idx_5hz_zstd7_EE_pxl_coords_expert_demos_imp_act +expert_dataset_dirpath: ${root_dir}/FISH/expert_demos/${suite.name}/${task_name}/${name_of_expert_demo} +expert_dataset: ${expert_dataset_dirpath}/demos.zarr +action_key: ${oc.if_else:${action_trajectories}, 'action_trajectory_${interpolation_frequency}hz', + 'action'} +semantic_demo_grouping_name: semantic_demo_grouping.yaml +semantic_demo_grouping: ${expert_dataset_dirpath}/${semantic_demo_grouping_name} +expert_dataset_config: ${expert_dataset_dirpath}/demo_config.yaml +bc_regularize: false +bc_weight_type: qfilter +load_checkpoint: ${agent.load_checkpoint} +wandb_run_id: '1013_1' +true_action_history: false +wandb_notes: null +checkpoint_epoch: 12000 +load_residual_weight: false +checkpoint_root_dir: /home/${oc.env:USER}/fish_leon/FISH +checkpoint_weight_dir: ${checkpoint_root_dir}/exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +residual_weight: ${root_dir}/FISH/weights/${suite.name}_${obs_type}/${task_name}/weight.pt +experiment_dir: ./exp_local/${suite.name}_${obs_type}/${task_name}/${wandb_run_id} +final_experiment_dir: ${experiment_dir}/${oc.generate_run_id:} +agent: + _target_: agent.diffusion_policy.DiffusionPolicyAgent + name: diffusion_policy + load_checkpoint: ${eval} + device: ${device} + n_obs_steps: ${.config.policy_cfg.n_obs_steps} + suite_name: ${suite.name} + obs_type: ${obs_type} + enable_arm: ${eval} + enable_camera: ${eval} + use_tb: ${use_tb} + desired_image_shape: + - 13 + - 180 + - 240 + orig_cam_shape: + - 3 + - 240 + - 320 + config: + _target_: agent.diffusion_policy.DiffusionPolicyAgentConfig + compile: false + device: ${device} + cam_resize_shape: ${agent.desired_image_shape} + orig_cam_shape: ${agent.orig_cam_shape} + policy_frequency: ${policy_frequency} + interpolation_frequency: ${interpolation_frequency} + policy_cfg: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.DiffusionConfig + n_obs_steps: 1 + horizon: 36 + n_action_steps: ${agent.config.policy_cfg.horizon} + input_shapes: + observation.image: ${agent.config.cam_resize_shape} + context_observation.image: ${agent.config.cam_resize_shape} + observation.state: + - 8 + observation.action_history: + - 7 + output_shapes: + action: + - 7 + input_normalization_modes: + observation.image: mean_std + observation.state: min_max + observation.action_history: min_max + output_normalization_modes: + action: min_max + vision_backbone: resnet18 + pretrained_backbone_weights: null + transforms: + - _target_: torchaug.transforms.RandomAffine + degrees: + - -5 + - 5 + translate: + - 0.05 + - 0.05 + batch_transform: true + num_chunks: -1 + batch_inplace: true + - _target_: torchaug.transforms.RandomColorJitter + brightness: 0.3 + contrast: 0.4 + saturation: 0.5 + hue: 0.08 + batch_transform: true + num_chunks: -1 + batch_inplace: true + use_group_norm: true + spatial_softmax_num_keypoints: 32 + action_history_encoder_config: + _target_: lerobot.common.policies.diffusion.configuration_diffusion.Unet1dEncoderConfig + in_channels: 7 + out_channels: 32 + history_length: ${agent.config.policy_cfg.n_action_steps} + kernel_size: ${agent.config.policy_cfg.kernel_size} + downsample_kernel_size: 3 + downsample_stride: 2 + downsample_padding: 1 + down_dims: + - 256 + - 512 + - 1024 + kernel_size: 5 + n_groups: 8 + diffusion_step_embed_dim: 128 + use_film_scale_modulation: true + noise_scheduler_type: DDIM + beta_schedule: squaredcos_cap_v2 + beta_start: 0.0001 + beta_end: 0.02 + prediction_type: epsilon + clip_sample: true + clip_sample_range: 1.0 + num_train_timesteps: 50 + num_inference_steps: 10 + do_mask_loss_for_padding: false + train_cfg: + _target_: utils.TrainConfig + lr: 0.0001 + lr_scheduler: cosine + lr_warmup_steps: 500 + adam_betas: + - 0.95 + - 0.999 + adam_eps: 1.0e-08 + adam_weight_decay: 1.0e-06 + grad_clip_norm: 10 + offline_steps: ${num_train_frames_diffusion} + use_amp: true + observation_cfg: + _target_: agent.encoder.VisualFeatureSet + use_depth: ${use_depth} + use_color: ${use_color} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_masks} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${color_crop_type} + depth_crop_type: ${depth_crop_type} + segmask_crop_type: ${segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${add_crop_binary_mask} + add_coord_conv_map: ${add_coord_conv_map} + context_input_config: + _target_: agent.encoder.ContextInputConfig + use_color: ${use_context_color} + use_depth: ${use_context_depth} + mask_input_dict: + _target_: agent.encoder.MaskInputDict + enable: ${use_context_segmask} + representation: ${mask_representation} + mask_list: ${mask_list} + crop_input_config: + _target_: agent.encoder.CropInputConfig + color_crop_type: ${context_color_crop_type} + depth_crop_type: ${context_depth_crop_type} + segmask_crop_type: ${context_segmask_crop_type} + crop_hw: ${crop_hw} + crop_down_offset: ${crop_down_offset} + add_crop_binary_mask: ${context_add_crop_binary_mask} + add_coord_conv_map: ${context_add_coord_conv_map} + mask_soft_approx_scheduler_config: + _target_: agent.encoder.MaskSoftApproxSchedulerConfig + num_steps: 40000 + initial_value: 10.0 + final_value: 1000.0 + interpolation_scheme: constant + use_contact_map: ${use_contact_map} + use_sdf_maps: ${use_sdf_maps} + use_normals_maps: ${use_normals_maps} + which_objects: ${which_objects} + grasped_dtc_max_value: ${grasped_dtc_max_value} + env_dtc_max_value: ${env_dtc_max_value} + grasped_normals_mask_max_dtc_value: ${grasped_normals_mask_max_dtc_value} + env_normals_mask_max_dtc_value: ${env_normals_mask_max_dtc_value} + clamp_dtc: ${clamp_dtc} + max_contact_prob: ${max_contact_prob} + mask_normals_within_sdf: ${mask_normals_within_sdf} + dtc_adaptive_normalization: ${dtc_adaptive_normalization} + adaptive_normals_mask: ${adaptive_normals_mask} + max_depth: ${max_depth} + image_shape: ${agent.desired_image_shape} + learnable_contact_preprocess_params: ${learnable_contact_preprocess_params} + learning_rate: ${agent.config.train_cfg.lr} + weight_decay: 0.0 + contact_model_name: ${contact_model_name} + zero_centered: false +suite: + suite: frankagym + name: frankagym + frame_stack: ${agent.n_obs_steps} + action_repeat: 1 + discount: 0.99 + hidden_dim: 1024 + num_train_frames: 2010 + num_seed_frames: 260 + num_train_epochs: 5000 + validate_every_epochs: 100 + validate_diffusion_on_action_loss_every_epochs: 500 + train_eval_diffusion_on_action_loss_every_epochs: 500 + check_topk_every_epochs: 10 + save_snapshot_every_epochs: 5000 + eval_every_frames: 2000 + num_eval_episodes: 5 + save_snapshot: true + wait_for_user_to_start_episode: true + task_make_fn: + _target_: suite.frankagym.make + name: ${task_name} + height: 240 + width: 320 + frame_stack: ${suite.frame_stack} + action_repeat: ${suite.action_repeat} + seed: ${seed} + enable_arm: ${agent.enable_arm} + enable_gripper: ${enable_gripper} + start_with_gripper_open: ${start_with_gripper_open} + enable_camera: ${agent.enable_camera} + path_to_depth_extrinsics: ${path_to_depth_extrinsics} + contact_estimation_model_ckpt_path: ${contact_estimation_model_ckpt_path} + x_limit: ${x_limit} + y_limit: ${y_limit} + z_limit: ${z_limit} + device: ${device} + interpolation_frequency: ${interpolation_frequency} + policy_frequency: ${policy_frequency} + debug_timestamps: ${debug_timestamps} + stop_after_action: ${stop_after_action} + open_loop: ${open_loop} + wait_for_new_camera_frames: ${wait_for_new_camera_frames} + action_key: ${action_key} + action_trajectory_horizon: ${agent.config.policy_cfg.horizon} + action_trajectories: ${action_trajectories} + path_to_zarr_dataset: ${expert_dataset} + agent_policy_cfg: ??? + true_action_history: ${true_action_history} +num_train_frames_bc: 50000 +num_train_frames_drq: 1100000 +stddev_schedule_drq: linear(1.0,0.1,100000) +task_name: FrankaInsertion-v1 +num_train_frames_vinn: 25000 +num_train_frames_diffusion: 1000000 +num_train_epochs_bc: 5000 +num_train_epochs_diffusion: 5000 +validate_every_epochs_bc: 5 +validate_every_epochs_diffusion: 25 +validate_diffusion_on_action_loss_every_epochs: 50 +train_eval_diffusion_on_action_loss_every_epochs: 500 +check_topk_every_epochs: 5 +check_topk_every_epochs_diffusion: ${validate_diffusion_on_action_loss_every_epochs} +save_snapshot_every_epochs_diffusion: 5000 +x_limit: +- 0.2 +- 0.7 +y_limit: +- -0.4 +- 0.4 +z_limit: +- -0.05 +- 0.55 +home_displacement: +- 0.55 +- 0.0 +- 0.55 +- 180.0 +- 0.0 +- 0.0 +enable_gripper: true +start_with_gripper_open: true +offset_mask: +- 1 +- 1 +- 1 +- 1 +- 1 +- 1 +path_to_depth_extrinsics: ~/fish_leon/FISH/cfgs/camera_poses/camera_poses_L515/20240904-122305/color_tf_world.npy diff --git a/x3d67guz/.hydra/hydra.yaml b/x3d67guz/.hydra/hydra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..956a565ed9acfe00a9b03fca5c220b619297b901 --- /dev/null +++ b/x3d67guz/.hydra/hydra.yaml @@ -0,0 +1,169 @@ +hydra: + run: + dir: ${final_experiment_dir} + sweep: + dir: ${final_experiment_dir} + subdir: ${hydra.job.num} + launcher: + submitit_folder: ${final_experiment_dir}/.slurm + timeout_min: 60 + cpus_per_task: null + gpus_per_node: null + tasks_per_node: 1 + mem_gb: null + nodes: 1 + name: ${hydra.job.name} + stderr_to_stdout: false + _target_: hydra_plugins.hydra_submitit_launcher.submitit_launcher.LocalLauncher + sweeper: + _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper + max_batch_size: null + params: null + help: + app_name: ${hydra.job.name} + header: '${hydra.help.app_name} is powered by Hydra. + + ' + footer: 'Powered by Hydra (https://hydra.cc) + + Use --hydra-help to view Hydra specific help + + ' + template: '${hydra.help.header} + + == Configuration groups == + + Compose your configuration from those groups (group=option) + + + $APP_CONFIG_GROUPS + + + == Config == + + Override anything in the config (foo.bar=value) + + + $CONFIG + + + ${hydra.help.footer} + + ' + hydra_help: + template: 'Hydra (${hydra.runtime.version}) + + See https://hydra.cc for more info. + + + == Flags == + + $FLAGS_HELP + + + == Configuration groups == + + Compose your configuration from those groups (For example, append hydra/job_logging=disabled + to command line) + + + $HYDRA_CONFIG_GROUPS + + + Use ''--cfg hydra'' to Show the Hydra config. + + ' + hydra_help: ??? + hydra_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][HYDRA] %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + root: + level: INFO + handlers: + - console + loggers: + logging_example: + level: DEBUG + disable_existing_loggers: false + job_logging: + version: 1 + formatters: + simple: + format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s' + handlers: + console: + class: logging.StreamHandler + formatter: simple + stream: ext://sys.stdout + file: + class: logging.FileHandler + formatter: simple + filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log + root: + level: INFO + handlers: + - console + - file + disable_existing_loggers: false + env: {} + mode: RUN + searchpath: [] + callbacks: {} + output_subdir: .hydra + overrides: + hydra: + - hydra.mode=RUN + task: + - agent=diffusion + - suite=frankagym + - suite/frankagym_task@_global_=insertion + job: + name: eval_policy + chdir: true + override_dirname: agent=diffusion,suite/frankagym_task@_global_=insertion,suite=frankagym + id: ??? + num: ??? + config_name: config_eval + env_set: {} + env_copy: [] + config: + override_dirname: + kv_sep: '=' + item_sep: ',' + exclude_keys: [] + runtime: + version: 1.3.2 + version_base: '1.1' + cwd: /home/leonmkim/fish_leon/FISH + config_sources: + - path: hydra.conf + schema: pkg + provider: hydra + - path: /home/leonmkim/fish_leon/FISH/cfgs + schema: file + provider: main + - path: '' + schema: structured + provider: schema + output_dir: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1013_1/x3d67guz + choices: + suite: frankagym + suite/frankagym_task@_global_: insertion + agent: diffusion + hydra/env: default + hydra/callbacks: null + hydra/job_logging: default + hydra/hydra_logging: default + hydra/hydra_help: default + hydra/help: default + hydra/sweeper: basic + hydra/launcher: submitit_local + hydra/output: default + verbose: false diff --git a/x3d67guz/.hydra/overrides.yaml b/x3d67guz/.hydra/overrides.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cb78bcdb3a5a10e7701d3b2e8898a2e783d11cc --- /dev/null +++ b/x3d67guz/.hydra/overrides.yaml @@ -0,0 +1,3 @@ +- agent=diffusion +- suite=frankagym +- suite/frankagym_task@_global_=insertion diff --git a/x3d67guz/eval_policy.log b/x3d67guz/eval_policy.log new file mode 100644 index 0000000000000000000000000000000000000000..41df623c309a798053ef2a043701568b52526e8a --- /dev/null +++ b/x3d67guz/eval_policy.log @@ -0,0 +1,15 @@ +[2025-01-06 20:10:51,091][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:428: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + @hydra.main(config_path='cfgs', config_name='config_eval') + +[2025-01-06 20:10:51,094][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:365: UserWarning: +The version_base parameter is not specified. +Please specify a compatability version level, or None. +Will assume defaults for version 1.1 + hydra.initialize( + +[2025-01-06 20:10:54,318][py.warnings][WARNING] - /home/leonmkim/fish_leon/FISH/eval_policy.py:414: FutureWarning: You are using `torch.load` with `weights_only=False` (the current default value), which uses the default pickle module implicitly. It is possible to construct malicious pickle data which will execute arbitrary code during unpickling (See https://github.com/pytorch/pytorch/blob/main/SECURITY.md#untrusted-models for more details). In a future release, the default value for `weights_only` will be flipped to `True`. This limits the functions that could be executed during unpickling. Arbitrary objects will no longer be allowed to be loaded via this mode unless they are explicitly allowlisted by the user via `torch.serialization.add_safe_globals`. We recommend you start setting `weights_only=True` for any use case where you don't have full control of the loaded file. Please open an issue on GitHub for any issues related to this experimental feature. + payload = torch.load(f) +