diff --git "a/7krw5tkq/wandb/debug-internal.log" "b/7krw5tkq/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/7krw5tkq/wandb/debug-internal.log" @@ -0,0 +1,892 @@ +2025-01-01 11:28:43,224 INFO StreamThr :1794543 [internal.py:wandb_internal():85] W&B internal server running at pid: 1794543, started at: 2025-01-01 11:28:43.224317 +2025-01-01 11:28:43,226 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status +2025-01-01 11:28:43,243 INFO WriterThread:1794543 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/run-7krw5tkq.wandb +2025-01-01 11:28:43,243 DEBUG SenderThread:1794543 [sender.py:send():379] send: header +2025-01-01 11:28:43,250 DEBUG SenderThread:1794543 [sender.py:send():379] send: run +2025-01-01 11:28:43,455 INFO SenderThread:1794543 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files +2025-01-01 11:28:43,455 INFO SenderThread:1794543 [sender.py:_start_run_threads():1188] run started: 7krw5tkq with start time 1735748923.224622 +2025-01-01 11:28:43,478 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: check_version +2025-01-01 11:28:43,478 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: check_version +2025-01-01 11:28:43,553 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: run_start +2025-01-01 11:28:43,557 DEBUG HandlerThread:1794543 [system_info.py:__init__():26] System info init +2025-01-01 11:28:43,558 DEBUG HandlerThread:1794543 [system_info.py:__init__():41] System info init done +2025-01-01 11:28:43,558 INFO HandlerThread:1794543 [system_monitor.py:start():194] Starting system monitor +2025-01-01 11:28:43,558 INFO SystemMonitor:1794543 [system_monitor.py:_start():158] Starting system asset monitoring threads +2025-01-01 11:28:43,558 INFO HandlerThread:1794543 [system_monitor.py:probe():214] Collecting system info +2025-01-01 11:28:43,558 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started cpu monitoring +2025-01-01 11:28:43,559 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started disk monitoring +2025-01-01 11:28:43,560 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started gpu monitoring +2025-01-01 11:28:43,560 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started memory monitoring +2025-01-01 11:28:43,561 INFO SystemMonitor:1794543 [interfaces.py:start():188] Started network monitoring +2025-01-01 11:28:43,573 DEBUG HandlerThread:1794543 [system_info.py:probe():152] Probing system +2025-01-01 11:28:43,574 DEBUG HandlerThread:1794543 [system_info.py:_probe_git():137] Probing git +2025-01-01 11:28:43,579 DEBUG HandlerThread:1794543 [system_info.py:_probe_git():145] Probing git done +2025-01-01 11:28:43,579 DEBUG HandlerThread:1794543 [system_info.py:probe():200] Probing system done +2025-01-01 11:28:43,579 DEBUG HandlerThread:1794543 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T16:28:43.573411', 'startedAt': '2025-01-01T16:28:43.213336', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3561.9015, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.45, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.001, 'min': 800.0, 'max': 4700.0}, {'current': 4500.013, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 704.9242362976074}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2025-01-01 11:28:43,579 INFO HandlerThread:1794543 [system_monitor.py:probe():224] Finished collecting system info +2025-01-01 11:28:43,579 INFO HandlerThread:1794543 [system_monitor.py:probe():227] Publishing system info +2025-01-01 11:28:43,579 DEBUG HandlerThread:1794543 [system_info.py:_save_code():44] Saving code +2025-01-01 11:28:43,581 DEBUG HandlerThread:1794543 [system_info.py:_save_code():67] Saving code done +2025-01-01 11:28:43,581 DEBUG HandlerThread:1794543 [system_info.py:_save_patches():84] Saving git patches +2025-01-01 11:28:43,645 DEBUG HandlerThread:1794543 [system_info.py:_save_patches():126] Saving git patches done +2025-01-01 11:28:43,645 INFO HandlerThread:1794543 [system_monitor.py:probe():229] Finished publishing system info +2025-01-01 11:28:43,648 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:28:43,648 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2025-01-01 11:28:43,649 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2025-01-01 11:28:43,649 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file diff.patch with policy now +2025-01-01 11:28:43,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: python_packages +2025-01-01 11:28:43,889 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: python_packages +2025-01-01 11:28:43,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:43,889 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:43,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:43,891 INFO HandlerThread:1794543 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/tb" + save: true +} +control { + relay_id: "c285d8b49483481285043490fdcd8b85" +} +_info { + stream_id: "7krw5tkq" +} + +2025-01-01 11:28:43,891 DEBUG HandlerThread:1794543 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2025-01-01 11:28:43,922 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/vrmuqfut-wandb-metadata.json +2025-01-01 11:28:43,926 INFO wandb-upload_1:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/fg6whylx-code/FISH/eval_robot.py +2025-01-01 11:28:43,947 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/ree151az-diff.patch +2025-01-01 11:28:43,977 DEBUG SenderThread:1794543 [sender.py:send():379] send: telemetry +2025-01-01 11:28:43,977 DEBUG SenderThread:1794543 [sender.py:send():379] send: tbrecord +2025-01-01 11:28:43,978 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:28:43,978 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 with policy live +2025-01-01 11:28:44,210 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/7m5mz9cw-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/diff.patch +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/requirements.txt +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/code/FISH/eval_robot.py +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-metadata.json +2025-01-01 11:28:44,457 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:28:44,458 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/code/FISH +2025-01-01 11:28:44,458 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/code +2025-01-01 11:28:44,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:45,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:46,456 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:28:46,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:47,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:48,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:48,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:49,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:50,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:51,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:52,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:53,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:53,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:54,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:55,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:56,190 DEBUG SenderThread:1794543 [sender.py:send():379] send: config +2025-01-01 11:28:56,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:57,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:58,459 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:28:58,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:28:58,889 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:28:58,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:28:58,964 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:28:59,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:00,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:01,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:02,461 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:02,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:03,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:04,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:04,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:05,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:06,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:07,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:08,462 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:08,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:09,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:10,013 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:10,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:11,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:12,463 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:12,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:13,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:13,889 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:13,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:14,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:15,025 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:15,464 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/config.yaml +2025-01-01 11:29:15,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:16,465 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:16,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:17,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:18,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:19,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:20,165 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:20,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:21,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:22,466 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:22,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:23,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:24,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:25,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:25,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:26,467 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:26,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:27,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:28,468 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:28,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:28,890 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:28,937 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:29,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:30,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:30,991 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:31,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:32,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:33,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:34,469 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:34,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:35,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:36,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:36,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:37,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:38,470 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:38,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:39,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:40,471 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:40,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:41,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:42,466 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:42,472 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:29:42,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:43,561 DEBUG SystemMonitor:1794543 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2025-01-01 11:29:43,562 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:29:43,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:43,890 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:43,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:44,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:45,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:46,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:47,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:48,002 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:48,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:49,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:50,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:51,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:52,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:53,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:53,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:54,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:55,899 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:56,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:57,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:58,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:29:58,890 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:29:58,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:29:59,015 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:29:59,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:00,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:01,900 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:02,901 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:03,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:04,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:04,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:05,902 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:06,902 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:07,902 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:08,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:09,132 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:30:09,132 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 with policy now +2025-01-01 11:30:09,132 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:30:09,133 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:30:09,133 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:30:09,135 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:30:09,404 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/ydfr40ny-media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos +2025-01-01 11:30:09,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:30:09,747 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/mtqwumgx-media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 +2025-01-01 11:30:09,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:10,146 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:10,479 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:10,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:11,903 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:12,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:13,562 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:30:13,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:13,891 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:13,937 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:14,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:15,193 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:15,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:16,481 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:16,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:17,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:18,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:19,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:20,249 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:20,482 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:20,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:21,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:22,483 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:22,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:23,905 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:24,484 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:24,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:25,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:25,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:26,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:27,485 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:30:27,735 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/zren7kmq-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:30:27,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:28,485 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:28,890 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:28,891 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:28,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:29,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:30,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:31,051 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:31,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:32,487 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:32,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:33,907 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:34,907 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:35,907 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:36,667 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:36,907 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:37,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:38,489 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:38,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:39,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:40,489 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:40,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:41,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:42,490 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:42,626 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:42,908 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:43,563 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:30:43,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:43,891 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:43,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:44,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:45,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:46,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:47,911 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:47,967 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:48,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:49,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:50,493 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:30:50,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:51,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:52,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:53,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:53,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:54,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:55,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:56,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:57,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:58,891 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:30:58,891 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:30:58,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:30:59,053 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:30:59,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:00,910 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:01,911 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:02,911 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:03,911 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:04,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:04,916 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:05,917 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:06,916 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:07,917 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:08,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:08,766 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:31:08,766 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 with policy now +2025-01-01 11:31:08,767 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:31:08,767 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:31:08,768 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:31:08,774 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:31:08,917 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:09,026 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/rs6n7a19-media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 +2025-01-01 11:31:09,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:31:09,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 +2025-01-01 11:31:09,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:31:09,785 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/yvpw51hj-media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 +2025-01-01 11:31:09,896 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:09,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:10,500 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:10,919 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:11,501 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:11,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:12,919 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:13,563 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:31:13,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:13,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:13,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:14,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:14,969 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:15,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:16,502 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:16,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:17,920 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:18,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:19,941 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:20,504 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:20,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:20,941 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:21,926 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:22,504 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:22,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:23,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:24,505 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:24,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:25,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:26,828 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:26,921 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:27,362 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:31:27,363 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:31:27,363 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:31:27,506 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:31:27,506 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:31:27,756 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/d9ukhhbb-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:31:27,922 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:28,507 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:28,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:28,892 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:28,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:29,922 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:30,922 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:31,852 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:31,923 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:32,927 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:33,923 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:34,509 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:34,926 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:35,924 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:36,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:36,924 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:37,924 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:38,510 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:38,924 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:39,925 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:40,511 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:40,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:41,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:41,981 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:42,511 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:42,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:43,564 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:31:43,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:43,909 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:43,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:44,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:45,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:46,513 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:31:46,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:47,929 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:47,981 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:48,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:49,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:50,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:51,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:52,930 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:53,897 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:53,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:54,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:55,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:56,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:57,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:58,892 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:31:58,893 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:31:58,933 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:31:59,000 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:31:59,931 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:00,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:01,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:02,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:03,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:04,898 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:04,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:05,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:06,932 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:07,938 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:08,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:08,935 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:08,955 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:32:08,955 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 with policy now +2025-01-01 11:32:09,001 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:32:09,002 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:32:09,002 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:32:09,005 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:32:09,289 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/hby0yuh8-media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 +2025-01-01 11:32:09,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:32:09,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 +2025-01-01 11:32:09,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:32:09,901 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/rwrh5u2l-media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 +2025-01-01 11:32:09,935 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:10,005 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:10,521 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:10,940 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:11,940 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:12,940 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:13,564 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:32:13,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:13,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:13,943 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:14,943 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:15,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:15,941 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:16,523 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:16,945 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:17,943 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:18,946 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:19,949 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:20,099 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:20,525 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:20,948 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:21,948 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:22,525 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:22,945 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:23,951 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:24,526 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:24,951 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:25,906 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:25,949 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:26,953 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:27,955 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:28,901 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:28,901 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:28,951 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:29,954 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:30,953 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:31,222 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:32:31,222 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:32:31,222 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:31,223 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:32:31,528 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:32:31,529 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:32:31,779 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/4rpyncsp-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:32:31,954 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:32,529 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:32,955 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:33,952 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:34,953 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:35,952 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:36,530 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:36,665 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:36,956 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:37,954 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:38,957 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:39,957 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:40,958 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:41,728 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:41,963 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:42,532 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:42,965 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:43,564 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:32:43,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:43,893 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:43,966 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:44,533 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:44,958 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:45,958 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:46,534 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:32:46,958 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:47,712 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:47,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:48,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:49,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:50,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:51,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:52,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:52,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:53,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:54,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:55,959 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:56,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:57,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:58,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:32:58,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:32:58,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:32:58,992 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:32:59,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:00,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:01,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:02,960 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:03,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:04,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:04,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:05,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:06,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:07,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:08,541 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:08,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:09,904 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:09,961 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:10,962 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:11,965 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:12,542 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:12,674 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:33:12,674 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 with policy now +2025-01-01 11:33:12,679 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:33:12,679 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:33:12,679 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:33:12,680 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:33:12,959 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/rbd0i1kg-media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 +2025-01-01 11:33:12,962 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:13,543 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:33:13,543 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 +2025-01-01 11:33:13,543 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:33:13,565 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:33:13,894 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/vv5utxx8-media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 +2025-01-01 11:33:13,902 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:13,902 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:13,966 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:14,543 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:14,973 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:15,025 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:15,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:16,973 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:17,977 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:18,544 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:18,977 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:19,970 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:20,914 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:20,980 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:21,972 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:22,974 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:23,976 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:24,546 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:24,974 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:25,975 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:26,547 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:26,801 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:26,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:27,977 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:28,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:28,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:28,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:29,978 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:30,978 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:31,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:32,295 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:33:32,295 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:33:32,295 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:32,296 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:33:32,549 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:32,549 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:33:32,549 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:33:32,820 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/v207z1qo-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:33:32,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:33,979 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:34,982 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:35,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:36,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:37,829 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:37,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:38,551 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:38,983 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:39,986 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:40,991 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:41,994 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:42,888 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:42,990 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:43,565 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:33:43,895 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:43,896 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:43,986 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:44,553 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:44,989 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:45,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:46,554 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:46,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:47,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:48,555 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:48,669 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:48,985 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:49,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:50,555 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:50,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:51,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:52,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:53,889 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:53,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:54,557 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:33:54,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:55,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:56,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:57,987 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:58,894 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:33:58,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:33:58,988 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:33:59,010 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:33:59,990 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:00,559 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:00,588 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:34:00,588 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 with policy now +2025-01-01 11:34:00,590 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:34:00,590 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:00,590 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:00,591 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:00,847 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/g48dsy20-media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 +2025-01-01 11:34:00,989 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:01,559 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:01,559 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 +2025-01-01 11:34:01,560 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval +2025-01-01 11:34:01,868 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/6e60bsk6-media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 +2025-01-01 11:34:01,993 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:02,560 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:02,995 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:03,560 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:03,994 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:04,915 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:05,003 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:06,001 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:06,561 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:07,008 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:08,000 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:09,009 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:09,922 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:10,003 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:11,004 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:12,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:12,563 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:13,007 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:13,566 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:34:13,901 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: stop_status +2025-01-01 11:34:13,901 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: stop_status +2025-01-01 11:34:14,005 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:15,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:15,713 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:16,015 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:16,564 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:17,017 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:18,018 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:19,015 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:19,350 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:19,350 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:19,350 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:19,565 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:19,566 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:34:20,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:20,048 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:34:20,049 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:34:20,212 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:34:20,566 INFO wandb-upload_2:1794543 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp6bulox_k +2025-01-01 11:34:20,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:20,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:34:20,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:34:20,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media +2025-01-01 11:34:20,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table +2025-01-01 11:34:20,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval +2025-01-01 11:34:20,589 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: log_artifact +2025-01-01 11:34:20,597 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: partial_history +2025-01-01 11:34:20,984 INFO SenderThread:1794543 [sender.py:send_request_log_artifact():1525] logged artifact run-7krw5tkq-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5MjI2MQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MjkzNg==', 'latestArtifact': None}} +2025-01-01 11:34:20,984 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:34:20,984 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json with policy now +2025-01-01 11:34:20,984 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:20,985 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:20,985 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:20,993 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:20,994 DEBUG SenderThread:1794543 [sender.py:send():379] send: telemetry +2025-01-01 11:34:20,994 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,008 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,008 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,009 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,009 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:21,025 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,025 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,026 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,026 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,047 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,047 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,048 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,048 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,069 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,070 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,070 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,070 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,088 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,089 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,107 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,107 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:21,124 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:21,124 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: log_artifact +2025-01-01 11:34:21,165 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/ktqrg3f1-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:34:21,407 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/r2jegu31-media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:34:21,569 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:21,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json +2025-01-01 11:34:21,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval +2025-01-01 11:34:21,604 INFO wandb-upload_0:1794543 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpnj7wzogk +2025-01-01 11:34:21,752 INFO wandb-upload_1:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/h1z6bubv-media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json +2025-01-01 11:34:22,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:22,019 INFO SenderThread:1794543 [sender.py:send_request_log_artifact():1525] logged artifact run-7krw5tkq-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDY5MjI3NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTI0MjkzOQ==', 'latestArtifact': None}} +2025-01-01 11:34:22,020 DEBUG SenderThread:1794543 [sender.py:send():379] send: files +2025-01-01 11:34:22,020 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json with policy now +2025-01-01 11:34:22,020 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:22,020 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:22,024 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:22,256 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/35ok2z6u-media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json +2025-01-01 11:34:22,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:22,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:23,012 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:23,334 DEBUG SenderThread:1794543 [sender.py:send():379] send: exit +2025-01-01 11:34:23,334 INFO SenderThread:1794543 [sender.py:send_exit():586] handling exit code: 0 +2025-01-01 11:34:23,334 INFO SenderThread:1794543 [sender.py:send_exit():588] handling runtime: 339 +2025-01-01 11:34:23,334 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:23,334 INFO SenderThread:1794543 [sender.py:send_exit():594] send defer +2025-01-01 11:34:23,334 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,334 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 0 +2025-01-01 11:34:23,335 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:23,335 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 0 +2025-01-01 11:34:23,335 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 1 +2025-01-01 11:34:23,335 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,335 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 1 +2025-01-01 11:34:23,335 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:23,335 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 1 +2025-01-01 11:34:23,335 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 2 +2025-01-01 11:34:23,335 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,335 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 2 +2025-01-01 11:34:23,335 INFO HandlerThread:1794543 [system_monitor.py:finish():203] Stopping system monitor +2025-01-01 11:34:23,335 DEBUG SystemMonitor:1794543 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2025-01-01 11:34:23,335 DEBUG SystemMonitor:1794543 [system_monitor.py:_start():183] Publishing last batch of metrics +2025-01-01 11:34:23,336 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined cpu monitor +2025-01-01 11:34:23,336 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined disk monitor +2025-01-01 11:34:23,387 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined gpu monitor +2025-01-01 11:34:23,388 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined memory monitor +2025-01-01 11:34:23,388 INFO HandlerThread:1794543 [interfaces.py:finish():200] Joined network monitor +2025-01-01 11:34:23,388 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:23,388 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 2 +2025-01-01 11:34:23,388 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 3 +2025-01-01 11:34:23,388 DEBUG SenderThread:1794543 [sender.py:send():379] send: stats +2025-01-01 11:34:23,389 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,389 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 3 +2025-01-01 11:34:23,389 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:23,389 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 3 +2025-01-01 11:34:23,389 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 4 +2025-01-01 11:34:23,389 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:23,389 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 4 +2025-01-01 11:34:23,570 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:24,571 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:30,356 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:30,356 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:30,356 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: keepalive +2025-01-01 11:34:30,357 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:30,357 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 4 +2025-01-01 11:34:30,357 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 5 +2025-01-01 11:34:30,357 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:30,358 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:30,358 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 5 +2025-01-01 11:34:30,358 DEBUG SenderThread:1794543 [sender.py:send():379] send: history +2025-01-01 11:34:30,358 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: summary_record +2025-01-01 11:34:30,359 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:30,359 DEBUG SenderThread:1794543 [sender.py:send():379] send: summary +2025-01-01 11:34:30,363 INFO SenderThread:1794543 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2025-01-01 11:34:30,363 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:30,363 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 5 +2025-01-01 11:34:30,363 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 6 +2025-01-01 11:34:30,363 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:30,363 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 6 +2025-01-01 11:34:30,363 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:30,363 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 6 +2025-01-01 11:34:30,367 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: status_report +2025-01-01 11:34:30,471 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 7 +2025-01-01 11:34:30,471 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:30,471 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 7 +2025-01-01 11:34:30,471 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:30,471 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 7 +2025-01-01 11:34:30,573 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/config.yaml +2025-01-01 11:34:30,573 INFO Thread-12 :1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:31,335 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:32,893 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 8 +2025-01-01 11:34:32,893 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:32,893 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:32,894 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 8 +2025-01-01 11:34:32,894 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:32,894 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 8 +2025-01-01 11:34:32,894 INFO SenderThread:1794543 [job_builder.py:build():440] Attempting to build job artifact +2025-01-01 11:34:32,896 INFO SenderThread:1794543 [job_builder.py:_get_source_type():569] is repo sourced job +2025-01-01 11:34:32,898 INFO SenderThread:1794543 [job_builder.py:build():545] adding wandb-job metadata file +2025-01-01 11:34:32,908 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 9 +2025-01-01 11:34:32,909 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:32,909 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 9 +2025-01-01 11:34:32,909 DEBUG SenderThread:1794543 [sender.py:send():379] send: artifact +2025-01-01 11:34:33,115 INFO SenderThread:1794543 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'state': 'COMMITTED', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDY2NzA0NA==', 'versionIndex': 404}}} +2025-01-01 11:34:33,115 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:33,115 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 9 +2025-01-01 11:34:33,115 INFO SenderThread:1794543 [dir_watcher.py:finish():358] shutting down directory watcher +2025-01-01 11:34:33,336 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:33,574 INFO SenderThread:1794543 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:33,575 INFO SenderThread:1794543 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files +2025-01-01 11:34:33,575 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/config.yaml config.yaml +2025-01-01 11:34:33,575 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json wandb-summary.json +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/requirements.txt requirements.txt +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-metadata.json wandb-metadata.json +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log output.log +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/diff.patch diff.patch +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2025-01-01 11:34:33,577 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 media/videos/eval/2_eval_2_b9fe5328ac66ca53d550.mp4 +2025-01-01 11:34:33,578 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 media/videos/eval/3_eval_3_5f29d76a7286353ca3a2.mp4 +2025-01-01 11:34:33,579 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 media/videos/eval/4_eval_4_b37fdb50af82045fc835.mp4 +2025-01-01 11:34:33,579 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 media/videos/eval/1_eval_1_6c62e0d922ed3f1dfaa8.mp4 +2025-01-01 11:34:33,580 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 media/videos/eval/0_eval_0_5cf24f1eb5bc7bb22047.mp4 +2025-01-01 11:34:33,583 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json media/table/eval/total_success_rate_ci_6_1b9c50b6c26dabbad645.table.json +2025-01-01 11:34:33,584 INFO SenderThread:1794543 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json media/table/eval/success_rate_ci_5_36776135f97d857e6145.table.json +2025-01-01 11:34:33,584 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 10 +2025-01-01 11:34:33,584 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:33,584 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:33,584 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 10 +2025-01-01 11:34:33,584 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:33,584 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 10 +2025-01-01 11:34:33,584 INFO SenderThread:1794543 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:34:33,762 INFO wandb-upload_0:1794543 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/wandb-summary.json +2025-01-01 11:34:33,775 INFO wandb-upload_1:1794543 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/config.yaml +2025-01-01 11:34:33,798 INFO wandb-upload_4:1794543 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/output.log +2025-01-01 11:34:33,831 INFO wandb-upload_3:1794543 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/files/requirements.txt +2025-01-01 11:34:33,855 INFO wandb-upload_2:1794543 [upload_job.py:push():130] Uploaded file /tmp/tmpj892nnbswandb/satjes7p-events.out.tfevents.1735748923.leonmkim-ROG-Strix-G15CS-G15CS.1794357.0 +2025-01-01 11:34:34,056 INFO Thread-11 (_thread_body):1794543 [sender.py:transition_state():614] send defer: 11 +2025-01-01 11:34:34,056 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:34,056 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 11 +2025-01-01 11:34:34,056 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:34,056 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 11 +2025-01-01 11:34:34,056 INFO SenderThread:1794543 [file_pusher.py:join():175] waiting for file pusher +2025-01-01 11:34:34,057 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 12 +2025-01-01 11:34:34,058 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:34,058 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 12 +2025-01-01 11:34:34,058 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:34,058 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 12 +2025-01-01 11:34:34,058 INFO SenderThread:1794543 [file_stream.py:finish():601] file stream finish called +2025-01-01 11:34:34,123 INFO SenderThread:1794543 [file_stream.py:finish():605] file stream finish is done +2025-01-01 11:34:34,123 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 13 +2025-01-01 11:34:34,123 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:34,124 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 13 +2025-01-01 11:34:34,124 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:34,124 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 13 +2025-01-01 11:34:34,124 INFO SenderThread:1794543 [sender.py:transition_state():614] send defer: 14 +2025-01-01 11:34:34,124 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: defer +2025-01-01 11:34:34,124 INFO HandlerThread:1794543 [handler.py:handle_request_defer():184] handle defer: 14 +2025-01-01 11:34:34,124 DEBUG SenderThread:1794543 [sender.py:send():379] send: final +2025-01-01 11:34:34,124 DEBUG SenderThread:1794543 [sender.py:send():379] send: footer +2025-01-01 11:34:34,124 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: defer +2025-01-01 11:34:34,125 INFO SenderThread:1794543 [sender.py:send_request_defer():610] handle sender defer: 14 +2025-01-01 11:34:34,125 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:34,125 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: server_info +2025-01-01 11:34:34,126 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: poll_exit +2025-01-01 11:34:34,126 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: get_summary +2025-01-01 11:34:34,127 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: sampled_history +2025-01-01 11:34:34,127 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:34,127 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: server_info +2025-01-01 11:34:34,127 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: internal_messages +2025-01-01 11:34:34,195 DEBUG SenderThread:1794543 [sender.py:send_request():406] send_request: poll_exit +2025-01-01 11:34:34,195 INFO MainThread:1794543 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2025-01-01 11:34:34,196 INFO MainThread:1794543 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2025-01-01 11:34:34,196 INFO MainThread:1794543 [wandb_run.py:_footer_sync_info():3975] logging synced files +2025-01-01 11:34:34,196 DEBUG HandlerThread:1794543 [handler.py:handle_request():158] handle_request: shutdown +2025-01-01 11:34:34,196 INFO HandlerThread:1794543 [handler.py:finish():882] shutting down handler +2025-01-01 11:34:35,127 INFO WriterThread:1794543 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1007_0/7krw5tkq/wandb/run-20250101_112843-7krw5tkq/run-7krw5tkq.wandb +2025-01-01 11:34:35,195 INFO SenderThread:1794543 [sender.py:finish():1615] shutting down sender +2025-01-01 11:34:35,195 INFO SenderThread:1794543 [file_pusher.py:finish():169] shutting down file pusher +2025-01-01 11:34:35,196 INFO SenderThread:1794543 [file_pusher.py:join():175] waiting for file pusher