diff --git "a/110822/wandb/debug-internal.log" "b/110822/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/110822/wandb/debug-internal.log" @@ -0,0 +1,758 @@ +2024-12-11 11:08:28,505 INFO StreamThr :736906 [internal.py:wandb_internal():85] W&B internal server running at pid: 736906, started at: 2024-12-11 11:08:28.505002 +2024-12-11 11:08:28,506 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status +2024-12-11 11:08:28,525 INFO WriterThread:736906 [datastore.py:open_for_write():87] open: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb +2024-12-11 11:08:28,525 DEBUG SenderThread:736906 [sender.py:send():379] send: header +2024-12-11 11:08:28,533 DEBUG SenderThread:736906 [sender.py:send():379] send: run +2024-12-11 11:08:28,813 INFO SenderThread:736906 [dir_watcher.py:__init__():211] watching files in: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files +2024-12-11 11:08:28,813 INFO SenderThread:736906 [sender.py:_start_run_threads():1188] run started: 7k8jf8l5 with start time 1733933308.5053 +2024-12-11 11:08:28,832 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: check_version +2024-12-11 11:08:28,833 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: check_version +2024-12-11 11:08:28,887 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: run_start +2024-12-11 11:08:28,891 DEBUG HandlerThread:736906 [system_info.py:__init__():26] System info init +2024-12-11 11:08:28,892 DEBUG HandlerThread:736906 [system_info.py:__init__():41] System info init done +2024-12-11 11:08:28,892 INFO HandlerThread:736906 [system_monitor.py:start():194] Starting system monitor +2024-12-11 11:08:28,892 INFO SystemMonitor:736906 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-12-11 11:08:28,892 INFO HandlerThread:736906 [system_monitor.py:probe():214] Collecting system info +2024-12-11 11:08:28,893 INFO SystemMonitor:736906 [interfaces.py:start():188] Started cpu monitoring +2024-12-11 11:08:28,893 INFO SystemMonitor:736906 [interfaces.py:start():188] Started disk monitoring +2024-12-11 11:08:28,894 INFO SystemMonitor:736906 [interfaces.py:start():188] Started gpu monitoring +2024-12-11 11:08:28,894 INFO SystemMonitor:736906 [interfaces.py:start():188] Started memory monitoring +2024-12-11 11:08:28,894 INFO SystemMonitor:736906 [interfaces.py:start():188] Started network monitoring +2024-12-11 11:08:28,905 DEBUG HandlerThread:736906 [system_info.py:probe():152] Probing system +2024-12-11 11:08:28,907 DEBUG HandlerThread:736906 [system_info.py:_probe_git():137] Probing git +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_info.py:_probe_git():145] Probing git done +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_info.py:probe():200] Probing system done +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_monitor.py:probe():223] {'os': 'Linux-5.15.0-125-generic-x86_64-with-glibc2.31', 'python': '3.10.14', 'heartbeatAt': '2024-12-11T16:08:28.905581', 'startedAt': '2024-12-11T16:08:28.497858', 'docker': None, 'cuda': None, 'args': ('agent=diffusion', 'suite=frankagym', 'suite/frankagym_task@_global_=insertion'), 'state': 'running', 'program': '/home/leonmkim/fish_leon/FISH/eval_robot.py', 'codePathLocal': None, 'codePath': 'FISH/eval_robot.py', 'git': {'remote': 'https://github.com/leonmkim/fish_leon.git', 'commit': '1b557f164c7f62a8d1545c761b13ede1677c3a83'}, 'email': 'leonmkim@seas.upenn.edu', 'root': '/home/leonmkim/fish_leon', 'host': 'leonmkim-ROG-Strix-G15CS-G15CS', 'username': 'leonmkim', 'executable': '/home/leonmkim/.pyenv/versions/lerobot/bin/python', 'cpu_count': 8, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3749.98675, 'min': 800.0, 'max': 4700.0}, 'cpu_freq_per_core': [{'current': 4499.916, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 3000.0, 'min': 800.0, 'max': 4700.0}, {'current': 4500.021, 'min': 800.0, 'max': 4700.0}, {'current': 4500.003, 'min': 800.0, 'max': 4700.0}], 'disk': {'/': {'total': 915.3232879638672, 'used': 623.7553443908691}}, 'gpu': 'NVIDIA GeForce RTX 2070 SUPER', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 2070 SUPER', 'memory_total': 8589934592}], 'memory': {'total': 62.71596908569336}} +2024-12-11 11:08:28,913 INFO HandlerThread:736906 [system_monitor.py:probe():224] Finished collecting system info +2024-12-11 11:08:28,913 INFO HandlerThread:736906 [system_monitor.py:probe():227] Publishing system info +2024-12-11 11:08:28,913 DEBUG HandlerThread:736906 [system_info.py:_save_code():44] Saving code +2024-12-11 11:08:28,915 DEBUG HandlerThread:736906 [system_info.py:_save_code():67] Saving code done +2024-12-11 11:08:28,915 DEBUG HandlerThread:736906 [system_info.py:_save_patches():84] Saving git patches +2024-12-11 11:08:28,980 DEBUG HandlerThread:736906 [system_info.py:_save_patches():126] Saving git patches done +2024-12-11 11:08:28,981 INFO HandlerThread:736906 [system_monitor.py:probe():229] Finished publishing system info +2024-12-11 11:08:28,984 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:08:28,984 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-metadata.json with policy now +2024-12-11 11:08:28,984 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file code/FISH/eval_robot.py with policy now +2024-12-11 11:08:28,984 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file diff.patch with policy now +2024-12-11 11:08:29,219 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: python_packages +2024-12-11 11:08:29,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: python_packages +2024-12-11 11:08:29,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:08:29,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:29,225 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:08:29,227 INFO HandlerThread:736906 [handler.py:handle_tbrecord():783] handling tbrecord: tbrecord { + log_dir: "/home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/tb" + save: true +} +control { + relay_id: "4910867fa5cc4067977e0abd74cdd7a1" +} +_info { + stream_id: "7k8jf8l5" +} + +2024-12-11 11:08:29,227 DEBUG HandlerThread:736906 [config_util.py:dict_from_config_file():70] no default config file found in config-defaults.yaml +2024-12-11 11:08:29,233 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/x1l1f2rp-wandb-metadata.json +2024-12-11 11:08:29,271 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/i9e9567q-diff.patch +2024-12-11 11:08:29,293 DEBUG SenderThread:736906 [sender.py:send():379] send: telemetry +2024-12-11 11:08:29,293 DEBUG SenderThread:736906 [sender.py:send():379] send: tbrecord +2024-12-11 11:08:29,294 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:08:29,294 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 with policy live +2024-12-11 11:08:29,297 INFO wandb-upload_1:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/rdhewrwt-code/FISH/eval_robot.py +2024-12-11 11:08:29,490 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/jpf62kj4-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:08:29,814 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-metadata.json +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/diff.patch +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH/eval_robot.py +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code +2024-12-11 11:08:29,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH +2024-12-11 11:08:30,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:31,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:31,814 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:31,970 DEBUG SenderThread:736906 [sender.py:send():379] send: config +2024-12-11 11:08:32,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:33,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:33,814 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:34,030 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:34,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:35,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:35,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:36,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:37,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:37,815 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:38,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:39,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:39,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:40,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:41,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:41,817 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:42,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:43,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:44,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:08:44,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:08:44,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:44,316 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:45,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:45,818 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:46,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:47,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:48,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:49,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:49,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:49,819 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:50,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:51,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:51,819 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:52,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:53,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:53,820 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:54,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:54,424 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:55,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:55,820 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:56,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:57,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:57,821 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:58,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:59,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:08:59,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:08:59,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:08:59,437 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:08:59,822 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:08:59,822 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml +2024-12-11 11:09:00,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:01,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:01,822 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:02,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:03,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:03,823 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:04,265 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:04,625 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:05,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:05,823 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:06,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:07,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:07,824 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:08,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:09,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:09,626 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:09,825 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:10,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:11,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:11,825 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:12,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:13,234 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:13,826 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:14,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:14,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:14,235 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:15,202 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:15,235 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:15,461 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:09:15,462 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:09:15,463 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 with policy now +2024-12-11 11:09:15,463 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:09:15,463 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:09:15,463 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:09:15,737 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/qmodnrad-media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:09:15,826 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos +2024-12-11 11:09:15,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:09:16,102 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/9jxf4xwe-media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:09:16,235 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:17,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:17,827 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:18,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:19,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:19,828 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:20,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:20,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:21,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:22,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:23,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:23,829 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:24,236 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:25,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:26,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:26,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:27,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:28,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:28,894 DEBUG SystemMonitor:736906 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-12-11 11:09:28,895 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:09:29,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:29,220 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:29,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:30,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:31,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:31,332 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:32,237 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:33,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:34,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:35,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:36,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:37,223 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:37,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:38,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:38,834 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:09:39,088 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/xszj87dx-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:09:39,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:39,835 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:40,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:41,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:41,835 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:42,238 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:42,452 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:43,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:43,836 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:44,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:44,221 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:44,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:45,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:45,837 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:46,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:47,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:47,554 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:47,838 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:48,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:49,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:49,838 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:50,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:51,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:51,839 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:52,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:52,750 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:53,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:53,840 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:54,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:55,240 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:55,840 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:56,241 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:57,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:57,841 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:09:57,956 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:09:58,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:58,895 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:09:59,220 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:09:59,221 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:09:59,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:09:59,842 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:00,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:01,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:01,843 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:02,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:03,148 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:03,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:03,843 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:04,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:05,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:05,844 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:06,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:07,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:07,845 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:08,244 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:08,347 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:09,245 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:09,845 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:10,245 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:11,251 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:11,425 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:10:11,425 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 with policy now +2024-12-11 11:10:11,426 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:10:11,426 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:10:11,427 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:10:11,429 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:10:11,681 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/vbziq3qb-media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:10:11,846 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:10:11,846 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:11,846 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:10:11,847 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:10:12,117 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/te2j2jvb-media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:10:12,253 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:13,255 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:13,444 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:13,847 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:14,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:14,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:14,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:15,256 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:15,847 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:16,258 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:17,253 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:17,848 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:18,256 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:19,074 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:19,254 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:20,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:21,258 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:22,256 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:23,257 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:24,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:24,257 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:25,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:25,624 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:10:25,624 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:10:25,625 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:10:25,851 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:10:25,851 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:10:26,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:27,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:27,851 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:28,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:28,896 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:10:29,221 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:29,222 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:29,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:29,343 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:29,852 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:30,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:31,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:31,852 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:32,259 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:33,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:33,853 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:34,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:34,484 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:35,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:35,854 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:36,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:37,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:37,855 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:38,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:39,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:39,684 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:39,855 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:40,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:41,260 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:41,856 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:42,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:43,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:43,857 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:44,222 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:44,222 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:44,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:44,710 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:45,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:45,858 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:46,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:47,261 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:47,858 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:48,265 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:49,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:49,859 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:49,896 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:50,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:51,262 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:51,860 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:52,263 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:53,263 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:53,861 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:54,263 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:55,085 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:10:55,264 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:55,861 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:56,264 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:57,264 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:57,862 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:10:58,266 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:58,595 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:10:58,595 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 with policy now +2024-12-11 11:10:58,597 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:10:58,598 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:10:58,598 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:10:58,598 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:10:58,862 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:10:58,863 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:10:58,863 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:10:58,869 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/u7agmibs-media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:10:58,896 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:10:59,153 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/veshwbiw-media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:10:59,224 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:10:59,225 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:10:59,270 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:10:59,863 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:00,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:00,392 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:01,267 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:02,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:03,269 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:03,864 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:04,305 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:05,278 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:05,865 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:06,273 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:06,306 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:07,276 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:08,277 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:09,275 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:10,274 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:11,277 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:12,232 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:12,278 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:13,274 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:14,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:14,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:14,277 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:15,281 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:16,280 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:17,282 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:17,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:17,475 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:11:17,475 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:11:17,475 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:11:17,868 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:11:17,869 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:11:18,141 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/wagm2b1h-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:11:18,283 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:19,288 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:19,869 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:20,287 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:21,285 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:21,870 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:22,285 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:22,473 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:23,282 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:23,871 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:24,301 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:25,284 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:25,871 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:26,282 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:27,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:27,477 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:27,872 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:28,283 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:28,897 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:11:29,227 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:29,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:29,283 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:29,873 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:30,285 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:31,284 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:31,873 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:32,284 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:32,506 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:33,286 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:33,874 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:34,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:35,288 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:35,875 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:36,291 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:37,291 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:37,511 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:37,875 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:38,289 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:39,288 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:39,876 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:40,290 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:41,290 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:41,877 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:42,291 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:42,593 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:43,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:43,878 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:44,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:44,226 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:44,293 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:45,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:45,880 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:46,292 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:47,294 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:47,631 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:47,880 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:48,293 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:49,329 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:49,881 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:50,298 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:51,303 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:51,882 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:52,314 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:52,331 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:11:52,331 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 with policy now +2024-12-11 11:11:52,377 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:11:52,379 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:11:52,379 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:11:52,380 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:11:52,610 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/yfw8lnde-media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:11:52,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:11:52,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:11:52,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:11:53,178 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/j207a1ci-media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:11:53,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:53,382 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:53,883 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:54,306 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:55,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:55,884 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:11:56,314 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:57,311 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:58,309 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:58,897 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:11:58,898 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:11:59,229 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:11:59,230 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:11:59,316 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:11:59,885 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:00,317 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:01,318 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:01,886 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:02,316 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:02,525 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:02,526 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:02,526 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:02,886 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:03,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:03,886 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:04,243 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:04,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:05,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:06,324 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:07,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:07,888 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:08,320 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:09,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:09,355 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:09,888 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:10,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:11,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:11,889 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:12,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:13,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:13,890 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:14,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:12:14,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:12:14,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:14,382 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:15,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:15,891 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:16,321 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:17,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:17,892 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:18,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:19,322 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:19,567 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:19,892 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:20,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:21,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:21,893 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:22,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:23,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:23,894 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:24,323 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:24,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:25,324 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:25,894 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:26,324 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:27,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:27,895 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:28,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:28,898 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:12:29,226 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:12:29,227 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:12:29,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:29,783 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:29,896 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:30,325 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:31,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:31,896 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:32,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:33,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:33,897 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:34,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:34,978 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:35,326 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:35,898 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:36,334 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:36,910 DEBUG SenderThread:736906 [sender.py:send():379] send: files +2024-12-11 11:12:36,910 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 with policy now +2024-12-11 11:12:36,913 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: partial_history +2024-12-11 11:12:36,913 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:36,913 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:36,914 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:37,174 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/5dcy7vyd-media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:37,337 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_created():271] file/dir created: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:37,899 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval +2024-12-11 11:12:38,192 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/663gsrmo-media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:38,346 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:39,333 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:40,341 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:40,890 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:41,334 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:41,900 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:42,334 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:43,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:43,901 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:44,239 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: stop_status +2024-12-11 11:12:44,239 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: stop_status +2024-12-11 11:12:44,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:45,348 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:45,454 DEBUG SenderThread:736906 [sender.py:send():379] send: exit +2024-12-11 11:12:45,454 INFO SenderThread:736906 [sender.py:send_exit():586] handling exit code: 0 +2024-12-11 11:12:45,454 INFO SenderThread:736906 [sender.py:send_exit():588] handling runtime: 256 +2024-12-11 11:12:45,488 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:send_exit():594] send defer +2024-12-11 11:12:45,489 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 0 +2024-12-11 11:12:45,489 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 1 +2024-12-11 11:12:45,489 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 1 +2024-12-11 11:12:45,489 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-12-11 11:12:45,489 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 2 +2024-12-11 11:12:45,489 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 2 +2024-12-11 11:12:45,489 INFO HandlerThread:736906 [system_monitor.py:finish():203] Stopping system monitor +2024-12-11 11:12:45,490 DEBUG SystemMonitor:736906 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-12-11 11:12:45,490 DEBUG SystemMonitor:736906 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-12-11 11:12:45,490 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined cpu monitor +2024-12-11 11:12:45,490 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined disk monitor +2024-12-11 11:12:45,539 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined gpu monitor +2024-12-11 11:12:45,540 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined memory monitor +2024-12-11 11:12:45,540 INFO HandlerThread:736906 [interfaces.py:finish():200] Joined network monitor +2024-12-11 11:12:45,540 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,540 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-12-11 11:12:45,540 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 3 +2024-12-11 11:12:45,540 DEBUG SenderThread:736906 [sender.py:send():379] send: stats +2024-12-11 11:12:45,540 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,541 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 3 +2024-12-11 11:12:45,541 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:45,541 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-12-11 11:12:45,541 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 4 +2024-12-11 11:12:45,541 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:45,541 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 4 +2024-12-11 11:12:45,901 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:45,902 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:45,902 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:46,159 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /tmp/tmpcaidxss4wandb/f6t033zr-events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:51,543 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: keepalive +2024-12-11 11:12:51,543 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-12-11 11:12:51,543 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:51,543 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 5 +2024-12-11 11:12:51,544 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:51,544 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:51,544 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:51,545 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 5 +2024-12-11 11:12:51,545 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:51,546 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:51,546 DEBUG SenderThread:736906 [sender.py:send():379] send: history +2024-12-11 11:12:51,546 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: summary_record +2024-12-11 11:12:51,549 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:51,550 DEBUG SenderThread:736906 [sender.py:send():379] send: summary +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:_save_file():1454] saving file wandb-summary.json with policy end +2024-12-11 11:12:51,550 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 6 +2024-12-11 11:12:51,550 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:51,550 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 6 +2024-12-11 11:12:51,550 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-12-11 11:12:51,550 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 7 +2024-12-11 11:12:51,551 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: status_report +2024-12-11 11:12:51,551 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:51,551 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 7 +2024-12-11 11:12:51,551 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:51,551 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-12-11 11:12:51,903 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:52,455 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:53,512 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 8 +2024-12-11 11:12:53,512 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:53,512 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:53,512 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 8 +2024-12-11 11:12:53,512 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:53,512 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-12-11 11:12:53,512 INFO SenderThread:736906 [job_builder.py:build():440] Attempting to build job artifact +2024-12-11 11:12:53,515 INFO SenderThread:736906 [job_builder.py:_get_source_type():569] is repo sourced job +2024-12-11 11:12:53,530 INFO SenderThread:736906 [job_builder.py:build():545] adding wandb-job metadata file +2024-12-11 11:12:53,545 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 9 +2024-12-11 11:12:53,545 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:53,545 DEBUG SenderThread:736906 [sender.py:send():379] send: artifact +2024-12-11 11:12:53,545 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 9 +2024-12-11 11:12:53,904 INFO Thread-12 :736906 [dir_watcher.py:_on_file_modified():288] file/dir modified: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:54,052 INFO wandb-upload_1:736906 [upload_job.py:push():85] Skipped uploading /home/leonmkim/.local/share/wandb/artifacts/staging/tmpym6m9k3x +2024-12-11 11:12:54,052 INFO wandb-upload_2:736906 [upload_job.py:push():85] Skipped uploading /tmp/tmpq1auzwlr/wandb-job.json +2024-12-11 11:12:54,169 INFO wandb-upload_0:736906 [upload_job.py:push():88] Uploaded file /home/leonmkim/.local/share/wandb/artifacts/staging/tmp9l1c58z3 +2024-12-11 11:12:54,456 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:54,800 INFO SenderThread:736906 [sender.py:send_artifact():1537] sent artifact job-https___github.com_leonmkim_fish_leon.git_FISH_eval_robot.py - {'id': 'QXJ0aWZhY3Q6MTM2NzU4MjI2NQ==', 'state': 'PENDING', 'artifactSequence': {'id': 'QXJ0aWZhY3RDb2xsZWN0aW9uOjMxNzk5NjkyNQ==', 'latestArtifact': {'id': 'QXJ0aWZhY3Q6MTM2NzU3NzcyOA==', 'versionIndex': 360}}} +2024-12-11 11:12:54,800 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:54,800 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-12-11 11:12:54,800 INFO SenderThread:736906 [dir_watcher.py:finish():358] shutting down directory watcher +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():388] scan: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml config.yaml +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json wandb-summary.json +2024-12-11 11:12:54,905 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt requirements.txt +2024-12-11 11:12:54,906 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-metadata.json wandb-metadata.json +2024-12-11 11:12:54,908 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 events.out.tfevents.1733933309.leonmkim-ROG-Strix-G15CS-G15CS.736760.0 +2024-12-11 11:12:54,909 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log output.log +2024-12-11 11:12:54,910 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/diff.patch diff.patch +2024-12-11 11:12:54,911 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/code/FISH/eval_robot.py code/FISH/eval_robot.py +2024-12-11 11:12:54,911 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 media/videos/eval/1_eval_1_dce37a039a0f3ae510e5.mp4 +2024-12-11 11:12:54,911 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 media/videos/eval/2_eval_2_c7cd658cf3e0e8f2dada.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 media/videos/eval/4_eval_4_b9f6613e7cd6916b7519.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 media/videos/eval/0_eval_0_a7ce057e6fe1a72a45a4.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [dir_watcher.py:finish():402] scan save: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 media/videos/eval/3_eval_3_ac712fc970ab6ee59b1c.mp4 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 10 +2024-12-11 11:12:54,912 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:54,912 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:54,912 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 10 +2024-12-11 11:12:54,912 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:54,912 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-12-11 11:12:54,912 INFO SenderThread:736906 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:12:55,091 INFO wandb-upload_2:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/requirements.txt +2024-12-11 11:12:55,126 INFO wandb-upload_1:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/config.yaml +2024-12-11 11:12:55,127 INFO wandb-upload_0:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/wandb-summary.json +2024-12-11 11:12:55,143 INFO wandb-upload_3:736906 [upload_job.py:push():130] Uploaded file /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/files/output.log +2024-12-11 11:12:55,344 INFO Thread-11 (_thread_body):736906 [sender.py:transition_state():614] send defer: 11 +2024-12-11 11:12:55,344 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,344 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 11 +2024-12-11 11:12:55,344 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,344 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-12-11 11:12:55,344 INFO SenderThread:736906 [file_pusher.py:join():175] waiting for file pusher +2024-12-11 11:12:55,345 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 12 +2024-12-11 11:12:55,345 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,345 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 12 +2024-12-11 11:12:55,345 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,345 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-12-11 11:12:55,346 INFO SenderThread:736906 [file_stream.py:finish():601] file stream finish called +2024-12-11 11:12:55,456 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:55,571 INFO SenderThread:736906 [file_stream.py:finish():605] file stream finish is done +2024-12-11 11:12:55,571 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 13 +2024-12-11 11:12:55,571 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:55,572 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,572 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 13 +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,572 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-12-11 11:12:55,572 INFO SenderThread:736906 [sender.py:transition_state():614] send defer: 14 +2024-12-11 11:12:55,572 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: defer +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send():379] send: final +2024-12-11 11:12:55,572 INFO HandlerThread:736906 [handler.py:handle_request_defer():184] handle defer: 14 +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send():379] send: footer +2024-12-11 11:12:55,572 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: defer +2024-12-11 11:12:55,573 INFO SenderThread:736906 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-12-11 11:12:55,573 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:55,573 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:55,573 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: server_info +2024-12-11 11:12:55,573 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: server_info +2024-12-11 11:12:55,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: poll_exit +2024-12-11 11:12:55,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: get_summary +2024-12-11 11:12:55,575 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: sampled_history +2024-12-11 11:12:55,576 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: internal_messages +2024-12-11 11:12:55,652 DEBUG SenderThread:736906 [sender.py:send_request():406] send_request: poll_exit +2024-12-11 11:12:55,652 INFO MainThread:736906 [wandb_run.py:_footer_history_summary_info():4016] rendering history +2024-12-11 11:12:55,652 INFO MainThread:736906 [wandb_run.py:_footer_history_summary_info():4048] rendering summary +2024-12-11 11:12:55,653 INFO MainThread:736906 [wandb_run.py:_footer_sync_info():3975] logging synced files +2024-12-11 11:12:55,653 DEBUG HandlerThread:736906 [handler.py:handle_request():158] handle_request: shutdown +2024-12-11 11:12:55,653 INFO HandlerThread:736906 [handler.py:finish():882] shutting down handler +2024-12-11 11:12:56,575 INFO WriterThread:736906 [datastore.py:close():296] close: /home/leonmkim/fish_leon/FISH/exp_local/frankagym_pixels/FrankaInsertion-v1/265644/110822/wandb/run-20241211_110828-7k8jf8l5/run-7k8jf8l5.wandb +2024-12-11 11:12:56,652 INFO SenderThread:736906 [sender.py:finish():1615] shutting down sender +2024-12-11 11:12:56,652 INFO SenderThread:736906 [file_pusher.py:finish():169] shutting down file pusher +2024-12-11 11:12:56,652 INFO SenderThread:736906 [file_pusher.py:join():175] waiting for file pusher