diff --git "a/203704/wandb/debug-internal.log" "b/203704/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/203704/wandb/debug-internal.log" @@ -0,0 +1,716 @@ +2024-12-17 20:37:10,293 INFO StreamThr :67053 [internal.py:wandb_internal():85] W&B internal server running at pid: 67053, started at: 2024-12-17 20:37:10.293327 +2024-12-17 20:37:10,296 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status +2024-12-17 20:37:10,306 INFO WriterThread:67053 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/run-rk784qf0.wandb +2024-12-17 20:37:10,307 DEBUG SenderThread:67053 [sender.py:send():379] send: header +2024-12-17 20:37:10,314 DEBUG SenderThread:67053 [sender.py:send():379] send: run +2024-12-17 20:37:10,558 INFO SenderThread:67053 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files +2024-12-17 20:37:10,558 INFO SenderThread:67053 [sender.py:_start_run_threads():1188] run started: rk784qf0 with start time 1734485830.293906 +2024-12-17 20:37:10,575 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: check_version +2024-12-17 20:37:10,575 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: check_version +2024-12-17 20:37:10,620 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: run_start +2024-12-17 20:37:10,628 DEBUG HandlerThread:67053 [system_info.py:__init__():26] System info init +2024-12-17 20:37:10,629 DEBUG HandlerThread:67053 [system_info.py:__init__():41] System info init done +2024-12-17 20:37:10,629 INFO HandlerThread:67053 [system_monitor.py:start():194] Starting system monitor +2024-12-17 20:37:10,629 INFO SystemMonitor:67053 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-17 20:37:10,630 INFO HandlerThread:67053 [system_monitor.py:probe():214] Collecting system info +2024-12-17 20:37:10,630 INFO SystemMonitor:67053 [interfaces.py:start():188] Started cpu monitoring +2024-12-17 20:37:10,631 INFO SystemMonitor:67053 [interfaces.py:start():188] Started disk monitoring +2024-12-17 20:37:10,631 INFO SystemMonitor:67053 [interfaces.py:start():188] Started gpu monitoring +2024-12-17 20:37:10,632 INFO SystemMonitor:67053 [interfaces.py:start():188] Started memory monitoring +2024-12-17 20:37:10,632 INFO SystemMonitor:67053 [interfaces.py:start():188] Started network monitoring +2024-12-17 20:37:10,643 DEBUG HandlerThread:67053 [system_info.py:probe():152] Probing system +2024-12-17 20:37:10,645 DEBUG HandlerThread:67053 [system_info.py:_probe_git():137] Probing git +2024-12-17 20:37:10,650 DEBUG HandlerThread:67053 [system_info.py:_probe_git():145] Probing git done +2024-12-17 20:37:10,650 DEBUG HandlerThread:67053 [system_info.py:probe():200] Probing system done +2024-12-17 20:37:10,650 DEBUG HandlerThread:67053 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-18T01:37:10.643921', 'startedAt': '2024-12-18T01:37:10.283939', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '5f28b54373d7d41e4ebace380ff1080a8ffc4051'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3936.407875, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.01, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4499.971, 'min': 800.0, 'max': 4700.0}, {'current': 4496.039, 'min': 800.0, 'max': 4700.0}, {'current': 4499.458, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 354.2344856262207}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-17 20:37:10,650 INFO HandlerThread:67053 [system_monitor.py:probe():224] Finished collecting system info +2024-12-17 20:37:10,650 INFO HandlerThread:67053 [system_monitor.py:probe():227] Publishing system info +2024-12-17 20:37:10,650 DEBUG HandlerThread:67053 [system_info.py:_save_code():44] Saving code +2024-12-17 20:37:10,652 DEBUG HandlerThread:67053 [system_info.py:_save_code():67] Saving code done +2024-12-17 20:37:10,652 DEBUG HandlerThread:67053 [system_info.py:_save_patches():84] Saving git patches +2024-12-17 20:37:10,751 DEBUG HandlerThread:67053 [system_info.py:_save_patches():126] Saving git patches done +2024-12-17 20:37:10,751 INFO HandlerThread:67053 [system_monitor.py:probe():229] Finished publishing system info +2024-12-17 20:37:10,754 DEBUG SenderThread:67053 [sender.py:send():379] send: files +2024-12-17 20:37:10,754 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-17 20:37:10,754 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-17 20:37:10,755 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-17 20:37:11,014 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: python_packages +2024-12-17 20:37:11,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:37:11,015 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: python_packages +2024-12-17 20:37:11,015 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:37:11,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:11,018 INFO HandlerThread:67053 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/tb" + save: true +} +control { + relay_id: "6b25a11cd35548b3a4b0bdd3bb1c8368" +} +_info { + stream_id: "rk784qf0" +} + +2024-12-17 20:37:11,018 DEBUG HandlerThread:67053 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-17 20:37:11,028 INFO wandb-upload_2:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/8wjdagy8-diff.patch +2024-12-17 20:37:11,028 INFO wandb-upload_1:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/r0rolyg1-code/FISH/eval_robot.py +2024-12-17 20:37:11,029 INFO wandb-upload_0:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/8w68qtgs-wandb-metadata.json +2024-12-17 20:37:11,070 DEBUG SenderThread:67053 [sender.py:send():379] send: telemetry +2024-12-17 20:37:11,083 DEBUG SenderThread:67053 [sender.py:send():379] send: tbrecord +2024-12-17 20:37:11,084 DEBUG SenderThread:67053 [sender.py:send():379] send: files +2024-12-17 20:37:11,084 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 with policy live +2024-12-17 20:37:11,264 INFO wandb-upload_1:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/kldg8w92-events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:37:11,559 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:37:11,560 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:11,560 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/code/FISH/eval_robot.py +2024-12-17 20:37:11,560 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/requirements.txt +2024-12-17 20:37:11,560 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-metadata.json +2024-12-17 20:37:11,560 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/diff.patch +2024-12-17 20:37:11,560 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/code/FISH +2024-12-17 20:37:11,560 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/code +2024-12-17 20:37:12,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:13,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:13,559 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:14,016 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:15,016 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:16,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:37:16,016 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:17,016 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:18,016 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:19,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:20,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:20,916 DEBUG SenderThread:67053 [sender.py:send():379] send: config +2024-12-17 20:37:21,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:37:21,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:22,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:23,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:23,561 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:24,018 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:25,018 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:26,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:37:26,015 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:37:26,018 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:26,150 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:37:27,018 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:27,563 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:28,018 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:29,018 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:29,563 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:30,018 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:31,018 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:31,524 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:37:32,019 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:33,019 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:33,564 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:34,019 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:35,019 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:36,019 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:36,591 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:37:37,019 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:37,565 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:38,019 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:39,019 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:39,566 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:40,020 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:41,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:37:41,015 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:37:41,020 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:41,567 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:41,647 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:37:42,020 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:42,567 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/config.yaml +2024-12-17 20:37:43,020 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:43,567 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:44,020 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:45,020 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:45,568 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:46,061 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:46,806 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:37:47,020 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:47,568 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:48,021 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:49,021 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:49,569 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:50,021 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:51,021 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:51,569 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:52,006 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:37:52,021 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:53,021 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:53,570 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:54,021 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:55,021 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:55,571 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:56,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:37:56,015 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:37:56,021 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:57,022 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:57,182 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:37:57,571 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:37:58,022 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:59,022 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:37:59,572 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:00,022 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:01,022 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:01,572 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:02,022 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:02,225 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:03,022 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:03,225 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 20:38:03,226 DEBUG SenderThread:67053 [sender.py:send():379] send: files +2024-12-17 20:38:03,226 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_b590c6d8e1bdcf382499.mp4 with policy now +2024-12-17 20:38:03,226 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:38:03,226 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:38:03,227 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:38:03,454 INFO wandb-upload_0:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/tk6lsq3o-media/videos/eval/0_eval_0_b590c6d8e1bdcf382499.mp4 +2024-12-17 20:38:03,573 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:03,573 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:38:03,573 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/0_eval_0_b590c6d8e1bdcf382499.mp4 +2024-12-17 20:38:03,574 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval +2024-12-17 20:38:03,574 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media +2024-12-17 20:38:03,574 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos +2024-12-17 20:38:03,836 INFO wandb-upload_1:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/d67d4dgz-media/videos/eval/0_eval_0_b590c6d8e1bdcf382499.mp4 +2024-12-17 20:38:04,025 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:05,025 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:05,574 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:06,025 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:07,023 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:07,349 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:07,574 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:08,023 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:09,065 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:10,023 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:10,633 DEBUG SystemMonitor:67053 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-17 20:38:10,633 DEBUG SenderThread:67053 [sender.py:send():379] send: stats +2024-12-17 20:38:11,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:38:11,016 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:38:11,057 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:11,576 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:38:11,576 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:11,807 INFO wandb-upload_0:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/kh3k3rv4-events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:38:12,023 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:12,401 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:13,024 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:13,577 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:14,024 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:15,024 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:16,024 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:17,024 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:17,578 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:17,600 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:18,024 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:19,024 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:19,579 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:20,027 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:21,027 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:21,579 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:22,027 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:22,605 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:23,027 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:23,580 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:24,027 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:25,027 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:25,581 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:26,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:38:26,015 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:38:26,027 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:27,031 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:27,581 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:27,605 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:28,028 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:29,029 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:29,582 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:30,029 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:31,030 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:31,583 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:32,030 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:32,801 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:33,030 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:33,583 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:34,030 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:35,031 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:35,584 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:36,031 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:37,032 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:37,585 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:37,998 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:38,032 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:39,033 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:39,586 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:40,032 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:40,634 DEBUG SenderThread:67053 [sender.py:send():379] send: stats +2024-12-17 20:38:41,015 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:38:41,016 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:38:41,032 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:41,586 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:42,032 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:43,033 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:43,259 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:43,305 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 20:38:43,306 DEBUG SenderThread:67053 [sender.py:send():379] send: files +2024-12-17 20:38:43,306 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_679fcb3e38369e1bb933.mp4 with policy now +2024-12-17 20:38:43,306 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:38:43,306 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:38:43,306 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:38:43,556 INFO wandb-upload_1:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/nn4fof7d-media/videos/eval/1_eval_1_679fcb3e38369e1bb933.mp4 +2024-12-17 20:38:43,587 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:38:43,587 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:43,587 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/1_eval_1_679fcb3e38369e1bb933.mp4 +2024-12-17 20:38:43,587 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval +2024-12-17 20:38:43,872 INFO wandb-upload_0:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/97jx77mj-media/videos/eval/1_eval_1_679fcb3e38369e1bb933.mp4 +2024-12-17 20:38:44,040 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:45,043 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:45,588 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:46,046 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:47,043 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:47,588 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:48,034 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:48,753 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:49,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:50,035 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:51,035 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:51,437 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:38:51,437 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:38:51,438 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:38:51,590 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:38:51,590 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:38:51,590 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:52,035 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:53,035 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:53,590 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:53,817 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:54,038 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:55,035 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:55,591 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:56,016 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:38:56,016 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:38:56,035 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:57,035 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:57,592 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:38:58,035 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:58,999 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:38:59,036 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:38:59,593 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:00,036 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:01,036 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:01,593 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:02,036 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:03,036 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:03,594 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:04,036 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:04,198 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:05,036 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:05,595 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:06,036 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:07,036 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:07,596 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:08,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:09,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:09,201 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:09,596 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:10,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:10,635 DEBUG SenderThread:67053 [sender.py:send():379] send: stats +2024-12-17 20:39:11,016 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:39:11,016 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:39:11,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:11,597 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:12,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:13,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:13,598 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:14,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:14,203 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:15,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:15,599 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:16,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:17,037 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:17,599 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:18,038 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:19,038 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:19,399 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:19,600 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:20,038 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:21,038 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:21,601 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:22,048 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:22,726 DEBUG SenderThread:67053 [sender.py:send():379] send: files +2024-12-17 20:39:22,726 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_06282d82872ca951a772.mp4 with policy now +2024-12-17 20:39:22,727 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 20:39:22,728 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:39:22,728 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:39:22,728 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:39:22,956 INFO wandb-upload_1:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/cvncpbbf-media/videos/eval/2_eval_2_06282d82872ca951a772.mp4 +2024-12-17 20:39:23,039 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:23,601 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:39:23,602 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:23,602 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/2_eval_2_06282d82872ca951a772.mp4 +2024-12-17 20:39:23,602 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval +2024-12-17 20:39:23,880 INFO wandb-upload_0:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/psyfnq41-media/videos/eval/2_eval_2_06282d82872ca951a772.mp4 +2024-12-17 20:39:24,040 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:24,757 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:25,046 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:26,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:39:26,017 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:39:26,048 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:27,052 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:27,603 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:28,050 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:29,042 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:29,603 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:29,805 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:30,049 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:30,272 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:39:30,272 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:39:30,272 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:39:30,604 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:39:30,604 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:39:30,825 INFO wandb-upload_1:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/bsq68gld-events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:39:31,048 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:31,604 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:32,046 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:33,051 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:33,605 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:34,052 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:34,943 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:35,051 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:35,606 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:36,051 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:37,075 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:37,606 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:38,051 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:39,051 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:39,607 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:39,981 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:40,051 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:40,636 DEBUG SenderThread:67053 [sender.py:send():379] send: stats +2024-12-17 20:39:41,016 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:39:41,016 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:39:41,051 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:41,608 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:42,052 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:43,052 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:43,609 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:44,052 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:45,052 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:45,178 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:45,610 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:46,052 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:47,052 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:47,611 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:48,052 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:49,053 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:49,611 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:50,053 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:50,374 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:51,053 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:51,612 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:52,053 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:53,053 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:53,613 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:54,053 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:55,053 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:55,572 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:39:55,614 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:56,016 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:39:56,016 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:39:56,054 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:57,054 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:57,615 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:39:58,054 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:59,054 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:39:59,615 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:00,054 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:00,769 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:01,062 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:01,616 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:02,063 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:02,086 DEBUG SenderThread:67053 [sender.py:send():379] send: files +2024-12-17 20:40:02,086 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_7e9bac7056f655aeeb9e.mp4 with policy now +2024-12-17 20:40:02,133 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 20:40:02,133 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:40:02,134 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:40:02,134 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:40:02,328 INFO wandb-upload_0:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/nutvs67j-media/videos/eval/3_eval_3_7e9bac7056f655aeeb9e.mp4 +2024-12-17 20:40:02,616 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:40:02,617 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/3_eval_3_7e9bac7056f655aeeb9e.mp4 +2024-12-17 20:40:02,617 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval +2024-12-17 20:40:02,888 INFO wandb-upload_1:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/urq2s85e-media/videos/eval/3_eval_3_7e9bac7056f655aeeb9e.mp4 +2024-12-17 20:40:03,061 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:03,617 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:04,063 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:05,067 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:05,926 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:06,065 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:07,066 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:07,618 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:08,063 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:09,062 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:09,619 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:10,063 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:10,637 DEBUG SenderThread:67053 [sender.py:send():379] send: stats +2024-12-17 20:40:11,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:40:11,017 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:40:11,065 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:11,164 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:12,069 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:13,070 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:14,072 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:15,072 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:16,066 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:17,032 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:17,070 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:18,068 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:19,072 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:19,326 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:40:19,326 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:40:19,327 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:40:19,622 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:40:19,622 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:40:20,077 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:21,076 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:21,623 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:22,035 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:22,077 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:23,074 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:23,624 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:24,073 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:25,073 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:25,624 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:26,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:40:26,017 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:40:26,073 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:27,073 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:27,220 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:27,625 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:28,074 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:29,074 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:29,626 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:30,074 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:31,074 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:31,627 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:32,074 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:32,411 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:33,074 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:33,627 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:34,074 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:35,075 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:35,628 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:36,075 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:37,075 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:37,608 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:37,629 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:38,075 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:39,075 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:39,630 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:40,075 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:40,637 DEBUG SenderThread:67053 [sender.py:send():379] send: stats +2024-12-17 20:40:41,017 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:40:41,017 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:40:41,075 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:41,630 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:42,075 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:42,808 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:43,076 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:43,631 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:44,076 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:45,076 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:45,632 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:46,076 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:47,076 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:47,633 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:47,827 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:48,076 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:49,076 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:49,633 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:50,076 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:51,084 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:51,552 DEBUG SenderThread:67053 [sender.py:send():379] send: files +2024-12-17 20:40:51,552 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_8b38e53a834b653d0330.mp4 with policy now +2024-12-17 20:40:51,553 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: partial_history +2024-12-17 20:40:51,553 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:40:51,553 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:40:51,555 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:40:51,634 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:40:51,634 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:51,634 INFO Thread-12 :67053 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/4_eval_4_8b38e53a834b653d0330.mp4 +2024-12-17 20:40:51,635 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval +2024-12-17 20:40:51,800 INFO wandb-upload_0:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/bsd0wy7p-media/videos/eval/4_eval_4_8b38e53a834b653d0330.mp4 +2024-12-17 20:40:52,077 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:52,091 INFO wandb-upload_1:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/bkuhxu1b-media/videos/eval/4_eval_4_8b38e53a834b653d0330.mp4 +2024-12-17 20:40:53,080 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:53,591 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:53,635 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:54,085 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:55,107 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:55,636 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:40:56,028 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: stop_status +2024-12-17 20:40:56,028 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: stop_status +2024-12-17 20:40:56,101 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:57,092 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:58,090 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:59,042 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:40:59,102 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:40:59,637 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:41:00,102 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:41:00,889 DEBUG SenderThread:67053 [sender.py:send():379] send: exit +2024-12-17 20:41:00,889 INFO SenderThread:67053 [sender.py:send_exit():586] handling exit code: 0 +2024-12-17 20:41:00,889 INFO SenderThread:67053 [sender.py:send_exit():588] handling runtime: 230 +2024-12-17 20:41:00,890 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:41:00,890 INFO SenderThread:67053 [sender.py:send_exit():594] send defer +2024-12-17 20:41:00,890 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:00,890 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-17 20:41:00,890 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:00,890 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-17 20:41:00,890 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 1 +2024-12-17 20:41:00,890 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:00,890 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-17 20:41:00,890 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:00,890 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-17 20:41:00,890 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 2 +2024-12-17 20:41:00,891 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:00,891 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-17 20:41:00,891 INFO HandlerThread:67053 [system_monitor.py:finish():203] Stopping system monitor +2024-12-17 20:41:00,891 DEBUG SystemMonitor:67053 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-17 20:41:00,891 INFO HandlerThread:67053 [interfaces.py:finish():200] Joined cpu monitor +2024-12-17 20:41:00,891 DEBUG SystemMonitor:67053 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-17 20:41:00,891 INFO HandlerThread:67053 [interfaces.py:finish():200] Joined disk monitor +2024-12-17 20:41:00,937 INFO HandlerThread:67053 [interfaces.py:finish():200] Joined gpu monitor +2024-12-17 20:41:00,937 INFO HandlerThread:67053 [interfaces.py:finish():200] Joined memory monitor +2024-12-17 20:41:00,937 INFO HandlerThread:67053 [interfaces.py:finish():200] Joined network monitor +2024-12-17 20:41:00,938 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:00,938 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-17 20:41:00,938 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 3 +2024-12-17 20:41:00,938 DEBUG SenderThread:67053 [sender.py:send():379] send: stats +2024-12-17 20:41:00,938 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:00,938 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-17 20:41:00,938 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:00,938 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-17 20:41:00,938 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 4 +2024-12-17 20:41:00,938 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:00,939 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-17 20:41:01,638 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:41:01,638 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:41:01,638 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:41:01,886 INFO wandb-upload_0:67053 [upload_job.py:push():130] Uploaded file /tmp/tmppmjusl6hwandb/mwblrpou-events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:41:06,940 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 20:41:06,941 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:41:06,941 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:06,941 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: keepalive +2024-12-17 20:41:06,941 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-17 20:41:06,941 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 5 +2024-12-17 20:41:06,941 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:41:06,941 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:06,941 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:41:06,941 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-17 20:41:06,942 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:41:06,942 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 20:41:06,942 DEBUG SenderThread:67053 [sender.py:send():379] send: history +2024-12-17 20:41:06,942 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: summary_record +2024-12-17 20:41:06,943 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:41:06,943 DEBUG SenderThread:67053 [sender.py:send():379] send: summary +2024-12-17 20:41:06,943 INFO SenderThread:67053 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-17 20:41:06,943 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:06,943 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-17 20:41:06,943 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 6 +2024-12-17 20:41:06,943 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:06,943 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-17 20:41:06,943 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:06,943 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-17 20:41:06,944 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 7 +2024-12-17 20:41:06,944 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: status_report +2024-12-17 20:41:06,944 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:06,944 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-17 20:41:06,944 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:06,944 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-17 20:41:07,640 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:41:07,890 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 20:41:09,435 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 8 +2024-12-17 20:41:09,435 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 20:41:09,435 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:09,436 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-17 20:41:09,436 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:09,436 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-17 20:41:09,436 INFO SenderThread:67053 [job_builder.py:build():440] Attempting to build job artifact +2024-12-17 20:41:09,438 INFO SenderThread:67053 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-17 20:41:09,453 INFO SenderThread:67053 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-17 20:41:09,465 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 9 +2024-12-17 20:41:09,465 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:09,465 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-17 20:41:09,465 DEBUG SenderThread:67053 [sender.py:send():379] send: artifact +2024-12-17 20:41:09,641 INFO Thread-12 :67053 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:41:09,891 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 20:41:09,983 INFO wandb-upload_2:67053 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp_8s9khff +2024-12-17 20:41:09,983 INFO wandb-upload_0:67053 [upload_job.py:push():85] Skipped uploading /tmp/tmpcadtafkn/wandb-job.json +2024-12-17 20:41:10,097 INFO wandb-upload_1:67053 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp5qyk3i_p +2024-12-17 20:41:10,801 INFO SenderThread:67053 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM4Mjk5MjQ0NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM4MjkyNjE2OQ==', 'versionIndex': 372}}} +2024-12-17 20:41:10,801 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:10,801 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-17 20:41:10,801 INFO SenderThread:67053 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-17 20:41:11,641 INFO SenderThread:67053 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files +2024-12-17 20:41:11,642 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/config.yaml config.yaml +2024-12-17 20:41:11,642 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json wandb-summary.json +2024-12-17 20:41:11,642 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/requirements.txt requirements.txt +2024-12-17 20:41:11,643 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-metadata.json wandb-metadata.json +2024-12-17 20:41:11,644 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 events.out.tfevents.1734485831.leonmkim-ROG-Strix-G15CS-G15CS.66864.0 +2024-12-17 20:41:11,646 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log output.log +2024-12-17 20:41:11,647 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/diff.patch diff.patch +2024-12-17 20:41:11,648 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-17 20:41:11,648 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/1_eval_1_679fcb3e38369e1bb933.mp4 media/videos/eval/1_eval_1_679fcb3e38369e1bb933.mp4 +2024-12-17 20:41:11,648 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/2_eval_2_06282d82872ca951a772.mp4 media/videos/eval/2_eval_2_06282d82872ca951a772.mp4 +2024-12-17 20:41:11,648 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/0_eval_0_b590c6d8e1bdcf382499.mp4 media/videos/eval/0_eval_0_b590c6d8e1bdcf382499.mp4 +2024-12-17 20:41:11,648 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/4_eval_4_8b38e53a834b653d0330.mp4 media/videos/eval/4_eval_4_8b38e53a834b653d0330.mp4 +2024-12-17 20:41:11,649 INFO SenderThread:67053 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/media/videos/eval/3_eval_3_7e9bac7056f655aeeb9e.mp4 media/videos/eval/3_eval_3_7e9bac7056f655aeeb9e.mp4 +2024-12-17 20:41:11,649 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 10 +2024-12-17 20:41:11,649 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 20:41:11,649 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:11,649 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-17 20:41:11,649 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:11,649 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-17 20:41:11,649 INFO SenderThread:67053 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 20:41:11,822 INFO wandb-upload_2:67053 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/config.yaml +2024-12-17 20:41:11,841 INFO wandb-upload_0:67053 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/requirements.txt +2024-12-17 20:41:11,843 INFO wandb-upload_1:67053 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/wandb-summary.json +2024-12-17 20:41:11,887 INFO wandb-upload_3:67053 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/files/output.log +2024-12-17 20:41:11,892 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 20:41:11,892 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 20:41:12,087 INFO Thread-11 (_thread_body):67053 [sender.py:transition_state():614] send defer: 11 +2024-12-17 20:41:12,088 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:12,088 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-17 20:41:12,088 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:12,088 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-17 20:41:12,088 INFO SenderThread:67053 [file_pusher.py:join():175] waiting for file pusher +2024-12-17 20:41:12,089 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 12 +2024-12-17 20:41:12,089 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:12,089 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-17 20:41:12,089 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:12,089 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-17 20:41:12,089 INFO SenderThread:67053 [file_stream.py:finish():601] file stream finish called +2024-12-17 20:41:12,148 INFO SenderThread:67053 [file_stream.py:finish():605] file stream finish is done +2024-12-17 20:41:12,148 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 13 +2024-12-17 20:41:12,148 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:12,149 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-17 20:41:12,149 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:12,149 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-17 20:41:12,149 INFO SenderThread:67053 [sender.py:transition_state():614] send defer: 14 +2024-12-17 20:41:12,149 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: defer +2024-12-17 20:41:12,149 INFO HandlerThread:67053 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-17 20:41:12,149 DEBUG SenderThread:67053 [sender.py:send():379] send: final +2024-12-17 20:41:12,149 DEBUG SenderThread:67053 [sender.py:send():379] send: footer +2024-12-17 20:41:12,149 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: defer +2024-12-17 20:41:12,149 INFO SenderThread:67053 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-17 20:41:12,150 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 20:41:12,150 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 20:41:12,150 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: server_info +2024-12-17 20:41:12,150 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: server_info +2024-12-17 20:41:12,150 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-17 20:41:12,152 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: get_summary +2024-12-17 20:41:12,152 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-17 20:41:12,152 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-17 20:41:12,216 DEBUG SenderThread:67053 [sender.py:send_request():406] send_request: poll_exit +2024-12-17 20:41:12,217 INFO MainThread:67053 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-17 20:41:12,217 INFO MainThread:67053 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-17 20:41:12,217 INFO MainThread:67053 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-17 20:41:12,217 DEBUG HandlerThread:67053 [handler.py:handle_request():158] handle_request: shutdown +2024-12-17 20:41:12,217 INFO HandlerThread:67053 [handler.py:finish():882] shutting down handler +2024-12-17 20:41:13,152 INFO WriterThread:67053 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/512_0/203704/wandb/run-20241217_203710-rk784qf0/run-rk784qf0.wandb +2024-12-17 20:41:13,217 INFO SenderThread:67053 [sender.py:finish():1615] shutting down sender +2024-12-17 20:41:13,217 INFO SenderThread:67053 [file_pusher.py:finish():169] shutting down file pusher +2024-12-17 20:41:13,217 INFO SenderThread:67053 [file_pusher.py:join():175] waiting for file pusher