diff --git "a/71icjjya/wandb/debug-internal.log" "b/71icjjya/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/71icjjya/wandb/debug-internal.log" @@ -0,0 +1,957 @@ +2024-12-31 23:52:32,529 INFO StreamThr :1770882 [internal.py:wandb_internal():85] W&B internal server running at pid: 1770882, started at: 2024-12-31 23:52:32.528874 +2024-12-31 23:52:32,531 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status +2024-12-31 23:52:32,545 INFO WriterThread:1770882 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/run-71icjjya.wandb +2024-12-31 23:52:32,545 DEBUG SenderThread:1770882 [sender.py:send():379] send: header +2024-12-31 23:52:32,552 DEBUG SenderThread:1770882 [sender.py:send():379] send: run +2024-12-31 23:52:32,774 INFO SenderThread:1770882 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files +2024-12-31 23:52:32,774 INFO SenderThread:1770882 [sender.py:_start_run_threads():1188] run started: 71icjjya with start time 1735707152.529045 +2024-12-31 23:52:32,789 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: check_version +2024-12-31 23:52:32,790 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: check_version +2024-12-31 23:52:32,836 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: run_start +2024-12-31 23:52:32,844 DEBUG HandlerThread:1770882 [system_info.py:__init__():26] System info init +2024-12-31 23:52:32,844 DEBUG HandlerThread:1770882 [system_info.py:__init__():41] System info init done +2024-12-31 23:52:32,845 INFO HandlerThread:1770882 [system_monitor.py:start():194] Starting system monitor +2024-12-31 23:52:32,845 INFO SystemMonitor:1770882 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-31 23:52:32,845 INFO HandlerThread:1770882 [system_monitor.py:probe():214] Collecting system info +2024-12-31 23:52:32,845 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started cpu monitoring +2024-12-31 23:52:32,846 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started disk monitoring +2024-12-31 23:52:32,847 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started gpu monitoring +2024-12-31 23:52:32,847 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started memory monitoring +2024-12-31 23:52:32,848 INFO SystemMonitor:1770882 [interfaces.py:start():188] Started network monitoring +2024-12-31 23:52:32,858 DEBUG HandlerThread:1770882 [system_info.py:probe():152] Probing system +2024-12-31 23:52:32,859 DEBUG HandlerThread:1770882 [system_info.py:_probe_git():137] Probing git +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_info.py:_probe_git():145] Probing git done +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_info.py:probe():200] Probing system done +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2025-01-01T04:52:32.858202', 'startedAt': '2025-01-01T04:52:32.521194', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': 'efd33a493dbad5ba8b9c770955ac9f0942bd971a'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.6215, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4497.485, 'min': 800.0, 'max': 4700.0}, {'current': 4499.702, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 621.5189628601074}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71595764160156}} +2024-12-31 23:52:32,864 INFO HandlerThread:1770882 [system_monitor.py:probe():224] Finished collecting system info +2024-12-31 23:52:32,864 INFO HandlerThread:1770882 [system_monitor.py:probe():227] Publishing system info +2024-12-31 23:52:32,864 DEBUG HandlerThread:1770882 [system_info.py:_save_code():44] Saving code +2024-12-31 23:52:32,866 DEBUG HandlerThread:1770882 [system_info.py:_save_code():67] Saving code done +2024-12-31 23:52:32,866 DEBUG HandlerThread:1770882 [system_info.py:_save_patches():84] Saving git patches +2024-12-31 23:52:32,927 DEBUG HandlerThread:1770882 [system_info.py:_save_patches():126] Saving git patches done +2024-12-31 23:52:32,928 INFO HandlerThread:1770882 [system_monitor.py:probe():229] Finished publishing system info +2024-12-31 23:52:32,931 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:52:32,931 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-31 23:52:32,932 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-31 23:52:32,932 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-31 23:52:33,166 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/5pwmmmvb-wandb-metadata.json +2024-12-31 23:52:33,167 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: python_packages +2024-12-31 23:52:33,167 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: python_packages +2024-12-31 23:52:33,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:52:33,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:33,169 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:52:33,171 INFO HandlerThread:1770882 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/tb" + save: true +} +control { + relay_id: "ed18d94dda074ba88639ea3a7fdd48f4" +} +_info { + stream_id: "71icjjya" +} + +2024-12-31 23:52:33,171 DEBUG HandlerThread:1770882 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-31 23:52:33,188 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/b3lguk4v-diff.patch +2024-12-31 23:52:33,223 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/vn55u65j-code/FISH/eval_robot.py +2024-12-31 23:52:33,229 DEBUG SenderThread:1770882 [sender.py:send():379] send: telemetry +2024-12-31 23:52:33,229 DEBUG SenderThread:1770882 [sender.py:send():379] send: tbrecord +2024-12-31 23:52:33,230 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:52:33,230 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 with policy live +2024-12-31 23:52:33,418 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ioz6e7gv-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:52:33,776 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:33,776 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:52:33,776 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-metadata.json +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH/eval_robot.py +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/diff.patch +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH +2024-12-31 23:52:33,777 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code +2024-12-31 23:52:34,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:35,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:35,775 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:36,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:37,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:38,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:38,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:39,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:40,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:41,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:42,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:43,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:43,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:44,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:45,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:45,872 DEBUG SenderThread:1770882 [sender.py:send():379] send: config +2024-12-31 23:52:46,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:47,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:47,779 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:48,167 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:52:48,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:52:48,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:48,235 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:49,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:50,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:51,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:52,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:53,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:53,780 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:54,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:54,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:55,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:56,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:57,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:57,781 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:52:58,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:59,169 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:52:59,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:52:59,782 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:00,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:01,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:01,783 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:02,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:03,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:03,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:03,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:04,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:04,224 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:04,784 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml +2024-12-31 23:53:05,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:05,784 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:06,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:07,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:07,785 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:08,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:09,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:09,517 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:09,785 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:10,172 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:11,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:12,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:13,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:14,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:15,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:15,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:16,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:17,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:18,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:18,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:18,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:19,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:20,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:20,301 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:21,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:21,789 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:22,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:23,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:23,789 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:24,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:25,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:25,759 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:26,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:27,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:27,790 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:28,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:29,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:29,791 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:30,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:30,766 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:31,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:31,791 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:53:32,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:32,848 DEBUG SystemMonitor:1770882 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-31 23:53:32,849 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:53:33,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:33,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:33,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:34,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:35,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:36,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:36,541 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:37,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:38,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:39,176 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:40,176 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:41,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:42,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:42,176 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:43,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:44,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:45,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:46,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:47,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:48,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:53:48,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:53:48,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:48,269 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:49,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:50,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:51,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:52,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:53,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:54,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:54,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:55,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:56,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:57,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:58,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:53:59,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:53:59,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:00,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:01,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:02,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:02,849 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:54:03,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:03,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:03,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:03,799 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:04,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:04,269 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:05,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:05,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:54:05,262 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:54:05,262 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 with policy now +2024-12-31 23:54:05,262 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:54:05,262 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:54:05,263 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:54:05,535 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/abdxmvd5-media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:54:05,800 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos +2024-12-31 23:54:05,801 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:54:05,801 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media +2024-12-31 23:54:06,089 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/26nhcuhz-media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:54:06,184 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:07,183 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:07,801 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:08,183 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:09,184 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:10,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:10,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:11,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:11,802 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:12,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:13,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:14,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:15,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:15,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:16,185 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:17,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:17,804 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:18,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:18,168 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:18,209 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:19,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:20,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:21,101 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:21,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:21,805 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:22,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:23,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:24,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:25,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:26,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:26,186 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:27,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:28,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:29,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:30,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:31,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:31,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:32,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:32,810 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:54:32,850 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:54:33,049 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/s91prls9-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:54:33,168 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:33,169 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:33,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:33,810 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:34,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:35,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:36,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:36,227 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:37,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:38,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:39,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:39,812 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:40,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:41,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:42,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:42,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:43,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:43,813 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:44,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:45,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:45,814 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:46,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:47,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:47,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:47,815 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:54:48,170 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:54:48,170 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:54:48,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:49,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:50,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:51,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:52,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:52,275 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:53,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:54,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:55,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:56,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:57,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:58,174 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:54:58,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:54:59,190 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:00,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:01,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:02,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:02,850 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:55:03,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:03,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:03,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:03,308 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:04,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:05,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:06,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:07,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:08,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:09,175 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:09,191 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:10,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:11,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:12,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:13,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:13,823 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:14,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:14,773 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:15,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:15,824 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:16,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:17,192 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:17,825 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:18,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:18,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:18,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:19,193 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:20,167 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:20,193 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:20,916 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:55:20,916 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 with policy now +2024-12-31 23:55:20,919 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:55:20,920 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:55:20,920 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:55:20,925 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:55:21,183 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/2citjgvy-media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:55:21,195 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:55:21,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:55:22,127 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/17gsyab6-media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:55:22,196 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:22,826 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:23,199 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:23,827 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:24,197 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:25,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:25,206 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:26,203 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:27,208 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:27,828 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:28,214 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:29,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:30,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:31,100 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:31,213 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:32,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:32,851 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:55:33,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:33,188 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:33,229 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:33,830 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:34,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:35,214 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:35,831 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:36,215 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:36,941 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:37,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:37,832 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:38,215 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:39,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:40,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:41,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:42,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:42,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:43,216 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:44,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:45,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:46,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:47,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:47,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:48,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:55:48,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:55:48,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:49,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:50,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:50,755 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:55:50,755 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:55:50,755 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:55:50,836 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:55:50,836 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:55:51,079 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/sq9owgqg-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:55:51,217 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:51,836 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:52,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:52,756 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:53,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:54,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:55,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:56,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:57,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:57,838 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:55:58,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:55:58,218 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:55:59,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:00,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:01,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:01,840 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:02,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:02,851 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:56:03,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:03,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:03,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:03,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:03,840 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:04,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:05,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:06,219 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:07,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:08,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:09,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:09,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:10,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:11,220 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:12,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:13,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:14,178 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:14,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:15,221 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:16,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:17,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:18,171 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:18,171 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:18,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:19,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:19,298 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:20,222 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:21,224 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:21,847 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:22,213 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:56:22,213 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 with policy now +2024-12-31 23:56:22,215 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:56:22,216 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:56:22,216 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:56:22,216 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:56:22,235 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:22,500 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/wkbtqql3-media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:56:22,848 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:22,848 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:56:22,849 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:56:22,849 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:56:23,129 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/kbw2oye5-media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:56:23,231 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:23,848 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:24,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:24,849 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:25,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:25,239 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:26,236 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:27,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:28,240 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:29,250 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:30,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:30,245 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:31,246 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:32,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:32,852 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:56:33,179 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:33,179 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:33,244 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:33,851 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:34,248 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:35,255 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:35,305 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:35,852 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:36,246 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:37,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:37,853 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:38,250 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:39,255 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:40,250 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:41,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:41,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:41,854 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:56:42,249 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:42,596 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:56:42,597 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:56:42,597 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:56:42,854 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:56:43,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:43,855 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:44,254 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:45,253 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:46,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:46,780 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:47,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:47,856 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:48,177 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:56:48,177 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:56:48,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:49,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:49,857 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:50,257 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:51,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:51,857 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:52,104 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:52,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:53,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:53,858 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:56:54,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:55,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:56,258 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:57,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:56:57,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:58,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:56:59,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:00,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:01,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:02,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:02,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:02,852 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:57:03,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:03,173 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:03,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:04,259 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:05,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:06,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:07,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:07,321 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:08,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:09,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:10,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:11,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:12,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:13,182 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:13,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:14,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:15,260 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:16,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:17,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:18,173 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:18,173 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:18,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:18,263 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:19,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:20,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:21,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:22,261 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:23,262 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:23,524 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:24,263 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:25,263 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:25,869 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:26,265 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:26,929 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:57:26,929 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 with policy now +2024-12-31 23:57:26,929 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:57:26,930 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:57:26,930 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:57:26,932 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:57:27,207 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/8o71j225-media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:57:27,268 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:27,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:27,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:57:27,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:57:27,871 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:57:28,164 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/e6w1qhze-media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:57:28,270 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:28,870 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:29,197 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:29,273 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:30,280 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:31,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:32,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:32,853 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:57:33,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:33,190 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:33,281 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:33,872 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:34,280 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:34,345 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:35,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:36,280 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:37,286 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:38,283 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:39,286 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:39,874 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:40,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:40,285 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:41,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:41,875 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:42,282 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:43,288 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:44,283 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:45,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:45,285 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:46,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:46,672 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:57:46,673 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:57:46,673 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:57:46,876 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:57:46,877 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:57:47,145 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/e7fk64i2-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:57:47,290 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:47,877 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:48,183 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:57:48,184 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:57:48,294 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:49,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:50,251 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:50,287 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:51,301 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:52,303 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:53,306 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:53,879 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:54,304 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:55,306 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:56,195 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:57:56,306 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:57,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:57,880 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:57:58,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:59,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:57:59,881 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:00,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:01,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:01,796 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:01,882 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:02,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:02,853 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:58:03,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:03,181 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:03,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:04,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:05,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:06,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:07,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:07,765 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:07,884 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:08,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:09,310 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:10,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:11,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:12,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:13,188 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:13,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:14,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:15,311 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:16,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:17,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:18,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:18,181 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:18,248 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:18,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:19,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:20,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:21,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:22,312 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:23,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:24,189 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:24,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:25,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:26,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:27,313 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:28,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:29,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:29,611 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:30,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:31,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:31,892 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:32,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:32,854 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:58:33,181 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:33,181 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:33,314 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:33,893 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:34,315 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:35,315 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:35,362 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:35,894 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:36,324 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:36,830 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:58:36,831 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 with policy now +2024-12-31 23:58:36,841 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:58:36,841 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:36,841 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:36,842 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:36,894 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:58:36,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:58:36,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval +2024-12-31 23:58:37,097 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ibli9dry-media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:58:37,316 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:37,364 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ijgpii4s-media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:58:37,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:38,323 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:38,895 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:39,326 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:40,325 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:41,205 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:41,321 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:42,331 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:43,335 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:43,897 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:44,341 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:45,339 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:46,208 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:46,337 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:47,335 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:47,898 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:48,187 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: stop_status +2024-12-31 23:58:48,187 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: stop_status +2024-12-31 23:58:48,338 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:49,344 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:49,899 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:50,345 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:51,348 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:51,900 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:51,977 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:58:52,353 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:53,364 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:54,347 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:54,751 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:54,751 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:54,752 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:54,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:58:54,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:58:55,170 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/mv7627av-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:58:55,297 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:58:55,298 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:58:55,347 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:58:55,435 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:58:55,717 INFO wandb-upload_1:1770882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpflqy7wn6 +2024-12-31 23:58:55,811 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: log_artifact +2024-12-31 23:58:55,819 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: partial_history +2024-12-31 23:58:55,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:58:55,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:58:55,901 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval +2024-12-31 23:58:55,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table +2024-12-31 23:58:56,070 INFO SenderThread:1770882 [sender.py:send_request_log_artifact():1525] logged artifact run-71icjjya-evalsuccess_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODQ5Mg==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDA0Ng==', 'latestArtifact': None}} +2024-12-31 23:58:56,071 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:58:56,071 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json with policy now +2024-12-31 23:58:56,071 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:56,071 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,072 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,072 DEBUG SenderThread:1770882 [sender.py:send():379] send: telemetry +2024-12-31 23:58:56,072 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,076 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,076 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,076 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,076 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,077 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,077 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,077 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,077 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,078 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,078 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,078 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,078 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,079 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,079 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,079 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,079 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,079 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,080 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,080 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,080 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,080 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,081 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: log_artifact +2024-12-31 23:58:56,254 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/ci4ufzv5-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:58:56,295 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/hbnryvaw-media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:58:56,422 INFO wandb-upload_1:1770882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp0_8k8kau +2024-12-31 23:58:56,486 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/urwqbvs2-media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:58:56,770 INFO SenderThread:1770882 [sender.py:send_request_log_artifact():1525] logged artifact run-71icjjya-evaltotal_success_rate_ci - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODQ5Nw==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjUyOTE4MDA0Nw==', 'latestArtifact': None}} +2024-12-31 23:58:56,770 DEBUG SenderThread:1770882 [sender.py:send():379] send: files +2024-12-31 23:58:56,770 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json with policy now +2024-12-31 23:58:56,770 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:58:56,770 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:58:56,771 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,771 DEBUG SenderThread:1770882 [sender.py:send():379] send: exit +2024-12-31 23:58:56,771 INFO SenderThread:1770882 [sender.py:send_exit():586] handling exit code: 0 +2024-12-31 23:58:56,771 INFO SenderThread:1770882 [sender.py:send_exit():588] handling runtime: 383 +2024-12-31 23:58:56,775 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:58:56,775 INFO SenderThread:1770882 [sender.py:send_exit():594] send defer +2024-12-31 23:58:56,775 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,775 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-31 23:58:56,776 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 1 +2024-12-31 23:58:56,776 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,776 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-31 23:58:56,776 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-31 23:58:56,776 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 2 +2024-12-31 23:58:56,776 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,776 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-31 23:58:56,776 INFO HandlerThread:1770882 [system_monitor.py:finish():203] Stopping system monitor +2024-12-31 23:58:56,776 DEBUG SystemMonitor:1770882 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-31 23:58:56,777 DEBUG SystemMonitor:1770882 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-31 23:58:56,777 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined cpu monitor +2024-12-31 23:58:56,777 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined disk monitor +2024-12-31 23:58:56,830 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined gpu monitor +2024-12-31 23:58:56,831 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined memory monitor +2024-12-31 23:58:56,831 INFO HandlerThread:1770882 [interfaces.py:finish():200] Joined network monitor +2024-12-31 23:58:56,831 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,831 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-31 23:58:56,831 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 3 +2024-12-31 23:58:56,831 DEBUG SenderThread:1770882 [sender.py:send():379] send: stats +2024-12-31 23:58:56,832 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,832 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-31 23:58:56,832 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:58:56,832 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-31 23:58:56,832 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 4 +2024-12-31 23:58:56,832 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:58:56,832 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-31 23:58:56,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:58:57,009 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/11xrgokq-media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:58:57,902 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: keepalive +2024-12-31 23:59:03,755 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,756 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-31 23:59:03,756 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 5 +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send():379] send: history +2024-12-31 23:59:03,756 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: summary_record +2024-12-31 23:59:03,757 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:03,759 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:59:03,760 DEBUG SenderThread:1770882 [sender.py:send():379] send: summary +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-31 23:59:03,760 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-31 23:59:03,760 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 6 +2024-12-31 23:59:03,760 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:03,760 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-31 23:59:03,761 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,761 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-31 23:59:03,761 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 7 +2024-12-31 23:59:03,761 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: status_report +2024-12-31 23:59:03,761 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:03,761 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-31 23:59:03,761 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:03,761 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-31 23:59:03,944 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:59:03,945 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml +2024-12-31 23:59:04,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:06,771 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 8 +2024-12-31 23:59:06,771 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:06,772 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:06,772 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-31 23:59:06,772 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:06,772 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-31 23:59:06,772 INFO SenderThread:1770882 [job_builder.py:build():440] Attempting to build job artifact +2024-12-31 23:59:06,774 INFO SenderThread:1770882 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-31 23:59:06,776 INFO SenderThread:1770882 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-31 23:59:06,790 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 9 +2024-12-31 23:59:06,790 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:06,790 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-31 23:59:06,791 DEBUG SenderThread:1770882 [sender.py:send():379] send: artifact +2024-12-31 23:59:06,946 INFO Thread-12 :1770882 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:59:07,112 INFO wandb-upload_1:1770882 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmp3bxld9c6 +2024-12-31 23:59:07,113 INFO wandb-upload_0:1770882 [upload_job.py:push():85] Skipped uploading /tmp/tmpw19jbu9c/wandb-job.json +2024-12-31 23:59:07,222 INFO wandb-upload_2:1770882 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmpvmqc2ow2 +2024-12-31 23:59:07,238 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:07,872 INFO SenderThread:1770882 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTQxNDIxODYyMQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTQxNDIwNzU2Mg==', 'versionIndex': 401}}} +2024-12-31 23:59:07,873 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:07,873 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-31 23:59:07,873 INFO SenderThread:1770882 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml config.yaml +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json wandb-summary.json +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt requirements.txt +2024-12-31 23:59:07,947 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-metadata.json wandb-metadata.json +2024-12-31 23:59:07,949 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:59:07,952 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log output.log +2024-12-31 23:59:07,953 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/diff.patch diff.patch +2024-12-31 23:59:07,953 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-31 23:59:07,956 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 media/videos/eval/3_eval_3_fe95f75420ab4bf1f2e0.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 media/videos/eval/0_eval_0_65829ddfa6c31573c9e7.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 media/videos/eval/2_eval_2_0389095f97a9dda10a03.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 media/videos/eval/4_eval_4_4c467f37c076e27eccc3.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 media/videos/eval/1_eval_1_45cd68318424e8a1aa34.mp4 +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json media/table/eval/total_success_rate_ci_6_f26da17f8eaa641b6744.table.json +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json media/table/eval/success_rate_ci_5_ff553b4df26f1e2b1b21.table.json +2024-12-31 23:59:07,957 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 10 +2024-12-31 23:59:07,957 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:07,957 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:07,957 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-31 23:59:07,958 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:07,958 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-31 23:59:07,958 INFO SenderThread:1770882 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:59:08,133 INFO wandb-upload_2:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/requirements.txt +2024-12-31 23:59:08,143 INFO wandb-upload_1:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/config.yaml +2024-12-31 23:59:08,161 INFO wandb-upload_0:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/wandb-summary.json +2024-12-31 23:59:08,185 INFO wandb-upload_4:1770882 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/files/output.log +2024-12-31 23:59:08,239 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:08,239 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:08,253 INFO wandb-upload_3:1770882 [upload_job.py:push():130] Uploaded file /tmp/tmp3283vy5dwandb/cehsxfet-events.out.tfevents.1735707153.leonmkim-ROG-Strix-G15CS-G15CS.1770473.0 +2024-12-31 23:59:08,453 INFO Thread-11 (_thread_body):1770882 [sender.py:transition_state():614] send defer: 11 +2024-12-31 23:59:08,453 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,453 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-31 23:59:08,454 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,454 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-31 23:59:08,454 INFO SenderThread:1770882 [file_pusher.py:join():175] waiting for file pusher +2024-12-31 23:59:08,455 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 12 +2024-12-31 23:59:08,455 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,455 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-31 23:59:08,455 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,455 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-31 23:59:08,455 INFO SenderThread:1770882 [file_stream.py:finish():601] file stream finish called +2024-12-31 23:59:08,638 INFO SenderThread:1770882 [file_stream.py:finish():605] file stream finish is done +2024-12-31 23:59:08,638 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 13 +2024-12-31 23:59:08,638 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,638 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,639 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-31 23:59:08,639 INFO SenderThread:1770882 [sender.py:transition_state():614] send defer: 14 +2024-12-31 23:59:08,639 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: defer +2024-12-31 23:59:08,639 INFO HandlerThread:1770882 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send():379] send: final +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send():379] send: footer +2024-12-31 23:59:08,639 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: defer +2024-12-31 23:59:08,639 INFO SenderThread:1770882 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:08,640 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: server_info +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-31 23:59:08,640 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: get_summary +2024-12-31 23:59:08,641 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-31 23:59:08,642 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: server_info +2024-12-31 23:59:08,643 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-31 23:59:08,694 DEBUG SenderThread:1770882 [sender.py:send_request():406] send_request: poll_exit +2024-12-31 23:59:08,695 INFO MainThread:1770882 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-31 23:59:08,695 INFO MainThread:1770882 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-31 23:59:08,695 INFO MainThread:1770882 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-31 23:59:08,695 DEBUG HandlerThread:1770882 [handler.py:handle_request():158] handle_request: shutdown +2024-12-31 23:59:08,696 INFO HandlerThread:1770882 [handler.py:finish():882] shutting down handler +2024-12-31 23:59:09,642 INFO WriterThread:1770882 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/1745_0/71icjjya/wandb/run-20241231_235232-71icjjya/run-71icjjya.wandb +2024-12-31 23:59:09,695 INFO SenderThread:1770882 [sender.py:finish():1615] shutting down sender +2024-12-31 23:59:09,695 INFO SenderThread:1770882 [file_pusher.py:finish():169] shutting down file pusher +2024-12-31 23:59:09,695 INFO SenderThread:1770882 [file_pusher.py:join():175] waiting for file pusher