diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,32 +1,32 @@ -2024-04-10 15:01:23,550 INFO StreamThr :100 [internal.py:wandb_internal():86] W&B internal server running at pid: 100, started at: 2024-04-10 15:01:23.549655 -2024-04-10 15:01:23,552 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status -2024-04-10 15:01:25,153 INFO WriterThread:100 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240410_150123-cuua991r/run-cuua991r.wandb -2024-04-10 15:01:25,153 DEBUG SenderThread:100 [sender.py:send():379] send: header -2024-04-10 15:01:25,156 DEBUG SenderThread:100 [sender.py:send():379] send: run -2024-04-10 15:01:25,650 INFO SenderThread:100 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240410_150123-cuua991r/files -2024-04-10 15:01:25,650 INFO SenderThread:100 [sender.py:_start_run_threads():1124] run started: cuua991r with start time 1712761283.550732 -2024-04-10 15:01:25,658 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: check_version -2024-04-10 15:01:25,658 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: check_version -2024-04-10 15:01:25,757 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: run_start -2024-04-10 15:01:25,769 DEBUG HandlerThread:100 [system_info.py:__init__():26] System info init -2024-04-10 15:01:25,769 DEBUG HandlerThread:100 [system_info.py:__init__():41] System info init done -2024-04-10 15:01:25,769 INFO HandlerThread:100 [system_monitor.py:start():194] Starting system monitor -2024-04-10 15:01:25,769 INFO SystemMonitor:100 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-10 15:01:25,769 INFO HandlerThread:100 [system_monitor.py:probe():214] Collecting system info -2024-04-10 15:01:25,770 INFO SystemMonitor:100 [interfaces.py:start():190] Started cpu monitoring -2024-04-10 15:01:25,771 INFO SystemMonitor:100 [interfaces.py:start():190] Started disk monitoring -2024-04-10 15:01:25,771 INFO SystemMonitor:100 [interfaces.py:start():190] Started gpu monitoring -2024-04-10 15:01:25,772 INFO SystemMonitor:100 [interfaces.py:start():190] Started memory monitoring -2024-04-10 15:01:25,773 INFO SystemMonitor:100 [interfaces.py:start():190] Started network monitoring -2024-04-10 15:01:25,787 DEBUG HandlerThread:100 [system_info.py:probe():150] Probing system -2024-04-10 15:01:25,789 DEBUG HandlerThread:100 [gitlib.py:_init_repo():56] git repository is invalid -2024-04-10 15:01:25,789 DEBUG HandlerThread:100 [system_info.py:probe():198] Probing system done -2024-04-10 15:01:25,789 DEBUG HandlerThread:100 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-10T15:01:25.787175', 'startedAt': '2024-04-10T15:01:23.542723', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '9088970cc926', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.184, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.184, 'min': 0.0, 'max': 0.0}, {'current': 2000.184, 'min': 0.0, 'max': 0.0}, {'current': 2000.184, 'min': 0.0, 'max': 0.0}, {'current': 2000.184, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5563.329231262207}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357559204101562}} -2024-04-10 15:01:25,789 INFO HandlerThread:100 [system_monitor.py:probe():224] Finished collecting system info -2024-04-10 15:01:25,789 INFO HandlerThread:100 [system_monitor.py:probe():227] Publishing system info -2024-04-10 15:01:25,789 DEBUG HandlerThread:100 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment -2024-04-10 15:01:26,652 INFO Thread-12 :100 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/conda-environment.yaml -2024-04-10 15:01:40,803 ERROR HandlerThread:100 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-04-10 15:56:50,267 INFO StreamThr :98 [internal.py:wandb_internal():86] W&B internal server running at pid: 98, started at: 2024-04-10 15:56:50.267018 +2024-04-10 15:56:50,269 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status +2024-04-10 15:56:51,016 INFO WriterThread:98 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240410_155650-albqrbhg/run-albqrbhg.wandb +2024-04-10 15:56:51,016 DEBUG SenderThread:98 [sender.py:send():379] send: header +2024-04-10 15:56:51,019 DEBUG SenderThread:98 [sender.py:send():379] send: run +2024-04-10 15:56:51,218 INFO SenderThread:98 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files +2024-04-10 15:56:51,218 INFO SenderThread:98 [sender.py:_start_run_threads():1124] run started: albqrbhg with start time 1712764610.268401 +2024-04-10 15:56:51,225 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: check_version +2024-04-10 15:56:51,226 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: check_version +2024-04-10 15:56:51,298 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: run_start +2024-04-10 15:56:51,309 DEBUG HandlerThread:98 [system_info.py:__init__():26] System info init +2024-04-10 15:56:51,309 DEBUG HandlerThread:98 [system_info.py:__init__():41] System info init done +2024-04-10 15:56:51,309 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 15:56:51,309 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 15:56:51,310 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 15:56:51,310 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 15:56:51,311 INFO HandlerThread:98 [system_monitor.py:probe():214] Collecting system info +2024-04-10 15:56:51,311 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 15:56:51,312 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 15:56:51,312 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 15:56:51,319 DEBUG HandlerThread:98 [system_info.py:probe():150] Probing system +2024-04-10 15:56:51,321 DEBUG HandlerThread:98 [gitlib.py:_init_repo():56] git repository is invalid +2024-04-10 15:56:51,321 DEBUG HandlerThread:98 [system_info.py:probe():198] Probing system done +2024-04-10 15:56:51,322 DEBUG HandlerThread:98 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-10T15:56:51.319241', 'startedAt': '2024-04-10T15:56:50.261059', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'ae604480b8c9', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.192, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.192, 'min': 0.0, 'max': 0.0}, {'current': 2000.192, 'min': 0.0, 'max': 0.0}, {'current': 2000.192, 'min': 0.0, 'max': 0.0}, {'current': 2000.192, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5563.648036956787}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357559204101562}} +2024-04-10 15:56:51,322 INFO HandlerThread:98 [system_monitor.py:probe():224] Finished collecting system info +2024-04-10 15:56:51,322 INFO HandlerThread:98 [system_monitor.py:probe():227] Publishing system info +2024-04-10 15:56:51,322 DEBUG HandlerThread:98 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-04-10 15:56:52,220 INFO Thread-12 :98 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/conda-environment.yaml +2024-04-10 15:57:06,336 ERROR HandlerThread:98 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda subprocess.call( @@ -37,427 +37,1425 @@ Traceback (most recent call last): File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait raise TimeoutExpired(self.args, timeout) subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds -2024-04-10 15:01:40,806 DEBUG HandlerThread:100 [system_info.py:_save_conda():222] Saving conda packages done -2024-04-10 15:01:40,806 INFO HandlerThread:100 [system_monitor.py:probe():229] Finished publishing system info -2024-04-10 15:01:40,813 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:01:40,813 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: keepalive -2024-04-10 15:01:40,813 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:01:40,813 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: keepalive -2024-04-10 15:01:40,813 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:01:40,813 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: keepalive -2024-04-10 15:01:40,814 DEBUG SenderThread:100 [sender.py:send():379] send: files -2024-04-10 15:01:40,814 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now -2024-04-10 15:01:41,539 INFO wandb-upload_0:100 [upload_job.py:push():131] Uploaded file /tmp/tmpy1um5qu9wandb/h830n11d-wandb-metadata.json -2024-04-10 15:01:41,655 INFO Thread-12 :100 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-metadata.json -2024-04-10 15:01:41,745 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: python_packages -2024-04-10 15:01:41,745 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: python_packages -2024-04-10 15:01:41,748 DEBUG SenderThread:100 [sender.py:send():379] send: telemetry -2024-04-10 15:01:41,749 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:01:41,759 DEBUG SenderThread:100 [sender.py:send():379] send: config -2024-04-10 15:01:41,799 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:01:41,800 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:01:41,800 DEBUG SenderThread:100 [sender.py:send():379] send: telemetry -2024-04-10 15:01:41,800 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:01:41,800 WARNING SenderThread:100 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-04-10 15:01:41,800 DEBUG SenderThread:100 [sender.py:send():379] send: telemetry -2024-04-10 15:01:41,800 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:01:42,656 INFO Thread-12 :100 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/requirements.txt -2024-04-10 15:01:43,992 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:01:48,993 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:01:53,999 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:01:54,660 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/config.yaml -2024-04-10 15:01:56,746 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:01:56,746 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:01:56,749 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:02:00,001 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:05,002 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:10,003 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:11,746 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:02:11,747 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:02:11,787 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:02:16,001 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:21,002 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:25,773 DEBUG SystemMonitor:100 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-10 15:02:25,775 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:02:26,746 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:02:26,747 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:02:26,787 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:02:26,936 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:31,937 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:36,754 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:02:36,756 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:36,757 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:36,757 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:36,757 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:36,757 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:02:36,757 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:02:36,759 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:02:37,675 INFO Thread-12 :100 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:02:37,760 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:39,609 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:02:39,612 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:39,612 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:39,613 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:39,613 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:39,613 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:39,613 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:39,613 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:39,613 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:02:39,613 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:02:39,614 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:02:39,614 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:02:39,676 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:02:39,676 INFO Thread-12 :100 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:02:41,677 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:02:41,880 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:02:41,880 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:02:41,921 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:02:43,090 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:48,091 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:53,092 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:55,775 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:02:56,753 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:02:56,753 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:02:56,766 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:02:58,966 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:02:59,684 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/config.yaml -2024-04-10 15:03:04,281 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:09,282 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:11,753 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:03:11,753 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:03:11,794 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:03:14,941 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:19,941 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:24,943 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:25,776 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:03:26,753 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:03:26,754 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:03:26,794 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:03:29,980 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:34,981 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:36,833 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:03:36,835 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:03:36,835 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:03:36,837 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:03:37,698 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:03:39,680 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:03:39,681 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:03:39,682 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:03:39,682 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:03:39,699 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:03:40,683 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:41,699 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:03:42,006 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:03:42,039 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:03:42,039 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:03:46,231 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:51,232 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:55,777 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:03:56,778 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:03:56,824 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:03:56,824 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:03:56,832 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:04:02,043 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:07,044 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:11,825 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:04:11,826 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:04:11,832 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:04:12,114 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:17,115 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:22,116 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:25,778 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:04:26,825 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:04:26,826 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:04:26,832 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:04:28,008 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:33,009 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:37,501 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:04:37,502 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:04:37,502 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:04:37,504 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:04:37,721 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:04:38,505 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:40,347 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:04:40,349 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:04:40,349 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:04:40,349 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:04:40,722 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:04:41,945 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:04:42,465 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:04:42,466 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:04:42,723 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:04:43,684 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:48,685 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:53,687 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:04:55,779 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:04:56,841 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:04:56,841 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:04:56,842 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:04:59,133 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:04,133 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:09,135 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:11,842 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:05:11,842 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:05:11,843 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:05:15,129 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:20,131 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:25,131 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:25,779 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:05:26,841 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:05:26,842 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:05:26,842 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:05:31,121 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:36,122 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:37,954 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:05:37,955 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:05:37,955 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:05:37,957 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:05:38,745 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:05:40,795 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:05:40,796 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:05:40,797 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:05:40,797 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:05:41,746 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:05:41,798 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:42,212 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:05:42,213 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:05:42,253 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:05:42,747 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:05:47,489 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:52,490 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:05:55,780 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:05:56,977 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:05:56,996 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:05:56,997 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:05:58,215 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:03,216 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:08,217 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:11,977 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:06:11,996 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:06:11,996 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:06:13,275 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:18,276 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:23,277 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:23,666 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: pause -2024-04-10 15:06:23,667 INFO HandlerThread:100 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-04-10 15:06:23,667 INFO HandlerThread:100 [system_monitor.py:finish():203] Stopping system monitor -2024-04-10 15:06:23,667 DEBUG SystemMonitor:100 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-04-10 15:06:23,667 DEBUG SystemMonitor:100 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-04-10 15:06:23,668 INFO HandlerThread:100 [interfaces.py:finish():202] Joined cpu monitor -2024-04-10 15:06:23,669 INFO HandlerThread:100 [interfaces.py:finish():202] Joined disk monitor -2024-04-10 15:06:23,673 INFO HandlerThread:100 [interfaces.py:finish():202] Joined gpu monitor -2024-04-10 15:06:23,674 INFO HandlerThread:100 [interfaces.py:finish():202] Joined memory monitor -2024-04-10 15:06:23,674 INFO HandlerThread:100 [interfaces.py:finish():202] Joined network monitor -2024-04-10 15:06:23,674 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:06:26,978 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:06:26,996 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:06:26,997 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:06:29,233 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:30,823 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: resume -2024-04-10 15:06:30,823 INFO HandlerThread:100 [handler.py:handle_request_resume():699] starting system metrics thread -2024-04-10 15:06:30,823 INFO HandlerThread:100 [system_monitor.py:start():194] Starting system monitor -2024-04-10 15:06:30,823 INFO SystemMonitor:100 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-10 15:06:30,824 INFO SystemMonitor:100 [interfaces.py:start():190] Started cpu monitoring -2024-04-10 15:06:30,825 INFO SystemMonitor:100 [interfaces.py:start():190] Started disk monitoring -2024-04-10 15:06:30,826 INFO SystemMonitor:100 [interfaces.py:start():190] Started gpu monitoring -2024-04-10 15:06:30,826 INFO SystemMonitor:100 [interfaces.py:start():190] Started memory monitoring -2024-04-10 15:06:30,827 INFO SystemMonitor:100 [interfaces.py:start():190] Started network monitoring -2024-04-10 15:06:32,242 DEBUG SenderThread:100 [sender.py:send():379] send: config -2024-04-10 15:06:32,243 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:06:32,243 DEBUG SenderThread:100 [sender.py:send():379] send: metric -2024-04-10 15:06:32,243 WARNING SenderThread:100 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-04-10 15:06:34,244 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:34,766 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:06:39,250 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:39,768 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/config.yaml -2024-04-10 15:06:41,978 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:06:41,996 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:06:41,997 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:06:45,276 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:50,277 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:55,277 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:06:56,977 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:06:56,996 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:06:56,997 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:07:01,236 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:06,237 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:11,238 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:11,985 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:07:11,997 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:07:11,997 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:07:17,194 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:22,195 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:26,698 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:07:26,699 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:07:26,699 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:07:26,700 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:07:26,786 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:07:26,982 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:07:26,997 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:07:26,997 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:07:27,274 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:29,557 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:07:29,558 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:07:29,559 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:07:29,559 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:07:29,787 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:07:30,828 DEBUG SystemMonitor:100 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-10 15:07:30,829 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:07:32,789 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:07:32,830 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:37,831 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:41,982 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:07:41,997 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:07:41,997 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:07:43,201 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:48,203 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:53,204 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:07:56,982 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:07:56,997 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:07:56,997 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:07:58,276 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:00,830 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:08:03,831 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:08,832 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:11,983 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:08:11,997 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:08:11,998 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:08:14,264 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:19,265 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:24,266 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:26,983 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:08:26,997 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:08:26,998 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:08:29,403 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:08:29,404 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:08:29,404 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:08:29,405 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:29,405 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:08:29,811 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:08:30,831 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:08:32,249 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:08:32,251 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:08:32,251 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:08:32,252 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:08:32,813 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:08:34,814 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:08:35,258 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:40,259 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:41,983 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:08:41,997 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:08:41,998 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:08:46,244 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:51,245 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:56,246 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:08:56,983 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:08:56,998 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:08:56,998 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:09:00,832 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:09:01,832 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:06,833 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:11,834 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:11,983 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:09:11,998 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:09:11,998 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:09:17,238 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:22,238 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:26,984 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:09:26,998 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:09:26,998 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:09:27,258 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:30,797 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:09:30,799 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:09:30,799 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:09:30,799 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:09:30,832 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:09:30,835 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:09:32,833 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:33,641 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:09:33,643 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:09:33,643 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:09:33,644 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:09:33,836 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:09:36,838 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:09:38,645 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:41,984 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:09:42,000 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:09:42,000 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:09:44,191 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:49,192 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:54,193 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:09:56,984 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:09:56,998 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:09:56,999 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:09:59,273 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:00,833 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:10:04,835 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:09,835 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:11,984 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:10:11,998 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:10:11,999 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:10:15,227 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:20,228 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:25,229 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:26,984 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:10:26,998 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:10:26,999 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:10:30,264 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:30,834 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:10:33,546 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:10:33,547 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:10:33,547 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:10:33,548 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:10:33,859 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:10:35,548 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:36,404 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:10:36,405 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:10:36,406 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:10:36,406 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:10:36,861 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:10:38,862 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:10:41,408 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:41,984 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:10:41,998 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:10:41,999 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:10:47,266 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:52,267 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:10:56,998 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:10:56,999 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:10:56,999 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:10:58,184 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:00,835 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:11:03,836 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:08,837 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:11,991 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:11:11,999 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:11:11,999 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:11:14,251 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:19,252 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:24,253 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:26,991 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:11:27,000 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:11:27,000 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:11:30,239 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:30,836 DEBUG SenderThread:100 [sender.py:send():379] send: stats -2024-04-10 15:11:34,816 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:11:34,817 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:11:34,818 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:11:34,818 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:11:34,883 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:11:35,819 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:37,666 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history -2024-04-10 15:11:37,667 DEBUG SenderThread:100 [sender.py:send():379] send: history -2024-04-10 15:11:37,668 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: summary_record -2024-04-10 15:11:37,668 INFO SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-10 15:11:37,884 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json -2024-04-10 15:11:38,885 INFO Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log -2024-04-10 15:11:41,669 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:41,991 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:11:42,000 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:11:42,000 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:11:47,192 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:52,193 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report -2024-04-10 15:11:56,991 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-10 15:11:57,000 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status -2024-04-10 15:11:57,000 DEBUG SenderThread:100 [sender.py:send_request():406] send_request: stop_status -2024-04-10 15:11:57,236 DEBUG HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:06,339 DEBUG HandlerThread:98 [system_info.py:_save_conda():222] Saving conda packages done +2024-04-10 15:57:06,339 INFO HandlerThread:98 [system_monitor.py:probe():229] Finished publishing system info +2024-04-10 15:57:06,346 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:06,346 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: keepalive +2024-04-10 15:57:06,346 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:06,347 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: keepalive +2024-04-10 15:57:06,347 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:06,347 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: keepalive +2024-04-10 15:57:06,347 DEBUG SenderThread:98 [sender.py:send():379] send: files +2024-04-10 15:57:06,347 INFO SenderThread:98 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-04-10 15:57:06,705 INFO wandb-upload_0:98 [upload_job.py:push():131] Uploaded file /tmp/tmpmiu24gukwandb/jxoeulwr-wandb-metadata.json +2024-04-10 15:57:07,223 INFO Thread-12 :98 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/wandb-metadata.json +2024-04-10 15:57:07,285 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: python_packages +2024-04-10 15:57:07,285 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: python_packages +2024-04-10 15:57:07,288 DEBUG SenderThread:98 [sender.py:send():379] send: telemetry +2024-04-10 15:57:07,298 DEBUG SenderThread:98 [sender.py:send():379] send: config +2024-04-10 15:57:07,298 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:57:07,299 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 15:57:07,300 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:57:07,302 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:57:07,448 DEBUG SenderThread:98 [sender.py:send():379] send: telemetry +2024-04-10 15:57:07,448 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 15:57:07,448 WARNING SenderThread:98 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-10 15:57:07,448 DEBUG SenderThread:98 [sender.py:send():379] send: telemetry +2024-04-10 15:57:08,224 INFO Thread-12 :98 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/requirements.txt +2024-04-10 15:57:10,449 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:15,450 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:20,456 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:21,229 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/config.yaml +2024-04-10 15:57:22,286 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:57:22,286 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:57:22,326 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:57:26,425 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:31,426 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:36,427 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:37,286 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:57:37,286 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:57:37,326 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:57:41,456 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:46,456 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:51,313 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 15:57:51,314 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 15:57:52,286 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:57:52,287 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:57:52,326 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:57:52,434 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:57:57,435 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:02,436 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:07,286 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:58:07,287 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:58:07,326 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:58:08,432 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:13,433 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:18,434 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:21,314 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 15:58:22,287 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:58:22,287 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:58:22,327 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:58:24,388 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:28,785 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 15:58:28,785 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 15:58:28,785 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 15:58:28,786 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 15:58:28,786 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 15:58:28,787 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 15:58:28,787 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 15:58:28,794 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 15:58:28,794 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 15:58:28,795 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 15:58:28,795 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 15:58:29,796 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:34,798 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:37,287 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:58:37,288 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:58:37,291 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:58:40,459 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:45,460 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:50,461 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:58:52,287 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:58:52,287 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:58:52,327 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:58:56,376 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:01,377 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:06,378 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:07,287 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:59:07,287 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:59:07,327 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:59:11,437 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:16,439 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:21,440 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:22,287 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:59:22,288 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:59:22,327 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:59:27,410 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:32,412 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:37,287 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:59:37,288 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:59:37,328 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:59:37,444 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:42,445 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:47,446 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:52,287 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 15:59:52,288 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 15:59:52,329 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 15:59:53,418 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 15:59:58,420 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:03,421 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:07,288 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:00:07,288 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:00:07,328 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:00:09,363 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:14,364 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:19,365 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:22,288 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:00:22,288 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:00:22,328 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:00:25,354 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:30,355 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:35,357 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:37,288 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:00:37,289 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:00:37,328 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:00:40,409 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:45,410 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:50,411 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:00:52,288 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:00:52,289 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:00:52,328 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:00:55,421 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:00,422 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:05,422 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:07,288 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:01:07,289 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:01:07,329 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:01:09,249 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:01:09,249 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:01:09,250 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:01:09,250 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:01:09,250 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:01:09,251 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:01:09,252 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:01:09,254 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:01:09,255 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:01:10,314 INFO Thread-12 :98 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:01:10,412 DEBUG SenderThread:98 [sender.py:send():379] send: config +2024-04-10 16:01:10,414 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:01:10,414 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:01:10,414 WARNING SenderThread:98 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-10 16:01:11,415 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:12,315 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:01:16,416 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:21,416 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:22,289 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:01:22,289 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:01:22,290 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:01:26,457 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:31,463 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:32,322 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/config.yaml +2024-04-10 16:01:36,584 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:37,289 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:01:37,289 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:01:37,329 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:01:42,425 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:47,425 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:52,289 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:01:52,290 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:01:52,329 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:01:53,385 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:01:58,386 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:03,387 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:05,704 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:02:05,704 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:02:05,704 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:02:05,704 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:02:05,704 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:02:05,704 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:02:05,706 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:02:05,706 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:02:05,712 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:02:05,712 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:02:05,712 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:02:05,713 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:02:07,289 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:02:07,290 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:02:07,290 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:02:08,456 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:13,457 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:15,609 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:02:15,609 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:02:15,609 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:02:15,610 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:02:15,610 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:02:15,611 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:02:15,612 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:02:15,612 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:02:15,614 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:02:15,649 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:02:15,649 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:02:15,649 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:02:15,650 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:02:15,650 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:02:15,650 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:02:15,650 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:02:15,650 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:02:15,655 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:02:15,655 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:02:15,655 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:02:15,655 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:02:18,656 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:22,289 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:02:22,290 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:02:22,290 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:02:24,396 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:29,397 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:34,398 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:37,289 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:02:37,290 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:02:37,330 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:02:40,390 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:45,392 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:50,392 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:52,290 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:02:52,290 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:02:52,331 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:02:54,085 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:02:54,086 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:02:54,086 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:02:54,086 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:02:54,086 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:02:54,087 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:02:54,088 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:02:54,090 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:02:54,091 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:02:55,207 DEBUG SenderThread:98 [sender.py:send():379] send: config +2024-04-10 16:02:55,209 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:02:55,209 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:02:55,209 WARNING SenderThread:98 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-10 16:02:56,209 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:02:56,352 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:03:01,211 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:06,217 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:06,374 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/config.yaml +2024-04-10 16:03:07,290 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:03:07,290 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:03:07,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:03:11,383 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:16,384 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:21,385 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:22,290 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:03:22,291 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:03:22,330 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:03:26,422 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:31,423 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:36,424 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:37,290 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:03:37,291 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:03:37,330 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:03:41,440 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:46,441 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:47,762 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: partial_history +2024-04-10 16:03:47,764 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:03:47,764 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:03:47,764 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:03:47,764 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:03:47,764 DEBUG SenderThread:98 [sender.py:send():379] send: history +2024-04-10 16:03:47,765 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: summary_record +2024-04-10 16:03:47,765 INFO SenderThread:98 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-10 16:03:48,389 INFO Thread-12 :98 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/wandb-summary.json +2024-04-10 16:03:51,766 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:03:52,290 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:03:52,291 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:03:52,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:03:53,537 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:03:53,537 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:03:53,537 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:03:53,537 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:03:53,537 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:03:53,538 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:03:53,539 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:03:53,539 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:03:53,544 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:03:53,544 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:03:53,544 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:03:53,545 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:03:57,546 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:02,547 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:07,290 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:04:07,291 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:04:07,293 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:04:08,406 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:09,396 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/config.yaml +2024-04-10 16:04:13,544 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:18,546 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:22,291 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:04:22,291 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:04:22,331 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:04:24,385 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:29,386 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:34,387 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:37,291 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:04:37,291 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:04:37,331 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:04:39,459 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:44,460 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:49,461 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:04:52,291 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:04:52,292 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:04:52,332 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:04:55,398 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:00,399 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:05,400 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:07,291 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:05:07,292 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:05:07,331 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:05:10,468 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:15,469 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:20,470 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:22,291 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:05:22,292 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:05:22,332 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:05:26,371 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:31,372 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:36,373 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:37,291 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:05:37,292 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:05:37,332 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:05:41,377 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:46,377 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:51,379 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:05:52,292 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:05:52,292 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:05:52,332 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:05:56,447 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:01,448 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:06,449 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:07,292 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:06:07,292 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:06:07,332 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:06:12,414 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:17,416 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:22,292 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:06:22,293 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:06:22,333 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:06:23,390 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:28,391 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:33,392 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:37,293 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:06:37,293 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:06:37,333 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:06:39,380 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:44,381 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:49,383 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:52,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:06:52,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:06:52,294 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:06:54,385 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:06:59,387 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:04,387 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:07,293 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:07:07,294 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:07:07,333 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:07:09,412 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:14,414 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:19,415 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:22,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:07:22,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:07:22,295 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:07:24,453 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:29,454 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:34,455 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:37,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:07:37,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:07:37,295 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:07:40,446 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:45,447 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:50,448 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:07:52,293 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:07:52,294 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:07:52,334 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:07:56,412 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:01,413 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:06,414 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:07,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:08:07,294 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:08:07,334 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:08:11,426 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:16,428 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:21,429 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:22,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:08:22,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:08:22,295 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:08:26,468 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:31,469 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:36,470 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:37,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:08:37,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:08:37,296 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:08:42,462 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:47,463 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:52,294 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:08:52,295 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:08:52,336 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:08:53,370 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:08:58,372 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:03,373 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:07,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:09:07,295 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:09:07,335 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:09:08,466 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:13,467 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:18,468 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:22,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:09:22,295 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:09:22,335 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:09:24,458 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:29,459 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:34,460 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:37,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:09:37,295 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:09:37,335 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:09:39,872 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:44,874 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:49,875 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:09:52,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:09:52,295 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:09:52,336 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:09:55,435 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:00,436 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:05,437 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:07,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:10:07,296 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:10:07,335 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:10:10,640 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:15,642 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:20,643 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:22,295 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:10:22,296 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:10:22,336 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:10:26,407 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:31,408 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:36,410 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:37,296 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:10:37,296 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:10:37,336 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:10:42,362 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:47,363 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:52,296 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:10:52,296 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:10:52,336 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:10:52,366 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:10:57,367 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:02,368 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:07,296 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:11:07,297 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:11:07,336 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:11:07,446 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:12,446 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:17,448 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:22,296 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:11:22,297 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:11:22,336 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:11:23,394 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:28,395 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:33,396 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:37,296 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:11:37,297 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:11:37,336 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:11:38,452 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:43,453 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:48,454 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:50,078 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:11:50,078 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:11:50,078 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:11:50,078 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:11:50,079 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:11:50,080 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:11:50,080 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:11:50,081 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:11:50,082 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:11:50,118 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:11:50,118 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:11:50,118 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:11:50,119 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:11:50,119 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:11:50,119 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:11:50,120 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:11:50,120 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:11:50,124 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:11:50,124 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:11:50,124 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:11:50,125 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:11:52,297 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:11:52,297 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:11:52,298 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:11:54,399 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:11:59,400 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:02,273 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:12:02,273 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:12:02,273 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:12:02,273 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:12:02,274 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:12:02,274 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:12:02,275 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:12:02,276 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:12:02,277 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:12:02,313 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:12:02,313 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:12:02,313 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:12:02,314 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:12:02,314 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:12:02,314 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:12:02,314 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:12:02,315 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:12:02,319 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:12:02,319 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:12:02,319 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:12:02,320 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:12:05,321 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:07,297 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:12:07,297 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:12:07,300 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:12:10,484 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:15,301 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:12:15,301 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:12:15,301 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:12:15,301 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:12:15,302 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:12:15,302 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:12:15,304 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:12:15,305 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:12:15,305 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:12:15,341 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:12:15,341 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:12:15,341 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:12:15,342 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:12:15,342 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:12:15,342 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:12:15,342 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:12:15,342 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:12:15,347 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:12:15,347 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:12:15,347 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:12:15,347 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:12:16,348 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:18,677 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:12:18,677 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:12:18,677 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:12:18,677 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:12:18,678 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:12:18,679 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:12:18,680 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:12:18,680 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:12:18,680 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:12:18,735 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:12:18,735 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:12:18,735 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:12:18,736 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:12:18,736 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:12:18,736 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:12:18,737 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:12:18,737 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:12:18,741 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:12:18,742 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:12:18,742 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:12:18,742 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:12:20,637 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:12:20,637 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:12:20,637 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:12:20,637 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:12:20,638 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:12:20,640 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:12:20,641 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:12:20,641 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:12:20,643 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:12:20,644 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:12:20,644 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:12:20,644 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:12:20,644 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:12:20,645 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:12:20,645 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:12:20,645 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:12:20,646 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:12:20,653 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:12:20,653 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:12:20,653 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:12:20,654 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:12:21,654 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:22,297 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:12:22,298 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:12:22,300 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:12:23,459 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:12:23,459 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:12:23,459 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:12:23,459 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:12:23,460 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:12:23,460 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:12:23,461 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:12:23,462 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:12:23,462 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:12:23,499 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:12:23,500 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:12:23,500 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:12:23,501 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:12:23,501 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:12:23,501 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:12:23,502 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:12:23,502 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:12:23,506 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:12:23,506 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:12:23,506 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:12:23,507 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:12:25,581 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:12:27,136 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:12:27,136 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:12:27,136 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:12:27,136 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:12:27,137 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:12:27,137 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:12:27,138 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:12:27,138 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:12:27,139 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:12:27,146 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:12:27,147 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:12:27,147 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:12:27,147 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:12:27,147 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:12:27,147 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:12:27,148 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:12:27,148 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:12:27,154 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:12:27,154 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:12:27,154 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:12:27,155 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:12:27,155 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:32,157 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:34,928 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:12:34,928 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:12:34,929 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:12:34,929 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:12:34,929 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:12:34,930 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:12:34,930 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:12:34,931 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:12:34,933 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:12:34,937 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:12:34,937 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:12:34,937 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:12:34,938 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:12:34,938 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:12:34,938 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:12:34,938 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:12:34,942 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:12:34,946 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:12:34,947 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:12:34,947 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:12:34,947 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:12:35,585 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:12:37,297 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:12:37,298 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:12:37,298 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:12:37,400 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:42,401 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:47,403 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:51,967 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:12:51,967 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:12:51,967 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:12:51,967 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:12:51,968 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:12:51,969 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:12:51,970 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:12:51,970 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:12:51,971 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:12:52,301 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:12:52,302 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:12:52,302 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:12:52,931 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:12:52,931 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:12:52,931 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:12:52,932 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:12:52,932 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:12:52,932 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:12:52,933 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:12:52,933 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:12:52,938 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:12:52,938 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:12:52,938 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:12:52,938 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:12:52,940 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:12:54,592 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:12:57,940 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:02,941 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:07,300 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:13:07,300 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:13:07,301 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:13:08,380 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:13,381 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:14,814 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:14,814 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:14,814 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:14,814 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:14,815 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:14,816 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:14,816 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:14,817 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:14,818 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:14,855 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:14,855 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:14,855 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:14,856 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:14,856 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:14,856 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:14,857 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:14,857 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:14,861 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:14,861 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:14,861 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:14,862 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:18,863 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:22,301 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:13:22,301 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:13:22,302 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:13:22,627 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:22,627 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:22,627 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:22,627 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:22,628 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:22,628 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:22,629 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:22,629 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:22,630 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:22,667 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:22,667 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:22,667 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:22,668 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:22,668 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:22,668 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:22,668 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:22,669 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:22,673 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:22,673 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:22,673 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:22,679 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:24,679 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:26,638 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:26,638 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:26,638 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:26,639 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:26,639 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:26,640 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:26,641 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:26,641 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:26,642 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:26,688 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:26,689 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:26,689 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:26,689 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:26,689 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:26,689 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:26,690 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:26,690 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:26,696 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:26,697 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:26,697 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:26,697 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:28,943 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:28,943 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:28,943 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:28,943 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:28,944 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:28,944 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:28,945 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:28,946 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:28,947 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:28,949 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:28,950 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:28,950 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:28,950 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:28,950 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:28,950 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:28,951 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:28,951 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:28,958 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:28,958 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:28,958 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:28,959 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:29,959 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:32,222 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:32,222 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:32,222 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:32,222 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:32,223 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:32,224 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:32,225 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:32,226 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:32,227 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:32,228 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:32,228 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:32,229 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:32,229 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:32,229 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:32,230 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:32,230 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:32,232 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:32,237 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:32,237 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:32,237 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:32,238 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:33,607 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:13:35,238 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:35,803 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:35,803 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:35,803 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:35,804 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:35,804 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:35,805 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:35,806 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:35,808 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:35,809 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:35,814 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:35,814 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:35,814 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:35,815 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:35,815 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:35,815 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:35,815 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:35,816 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:35,821 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:35,821 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:35,821 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:35,822 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:37,300 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:13:37,300 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:13:37,301 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:13:38,311 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:38,311 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:38,311 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:38,311 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:38,312 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:38,313 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:38,314 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:38,317 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:38,317 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:38,351 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:38,352 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:38,352 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:38,352 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:38,353 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:38,353 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:38,353 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:38,353 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:38,357 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:38,358 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:38,358 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:38,358 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:39,610 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:13:40,359 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:43,154 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:43,154 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:43,155 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:43,155 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:43,155 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:43,156 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:43,157 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:43,157 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:43,161 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:44,089 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:44,089 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:44,089 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:44,089 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:44,090 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:44,090 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:44,090 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:44,090 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:44,096 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:44,096 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:44,096 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:44,097 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:44,612 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:13:46,098 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:46,904 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:46,905 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:46,905 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:46,905 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:46,905 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:46,906 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:46,907 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:46,907 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:46,908 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:46,927 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:46,927 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:46,927 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:46,927 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:46,927 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:46,928 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:46,928 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:46,928 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:46,933 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:46,933 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:46,933 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:46,934 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:48,881 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:48,881 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:48,881 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:48,881 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:48,882 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:48,883 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:48,883 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:48,885 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:48,886 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:48,892 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:48,892 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:48,892 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:48,892 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:48,892 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:48,893 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:48,894 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:48,894 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:48,899 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:48,900 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:48,900 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:48,900 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:51,329 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:51,329 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:51,329 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:51,330 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:51,330 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:51,334 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:51,335 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:51,335 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:51,336 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:51,353 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:13:51,353 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:51,353 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:51,354 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:51,355 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:51,355 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:51,356 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:51,356 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:51,356 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:51,360 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:51,361 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:51,361 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:51,361 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:52,300 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:13:52,301 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:13:52,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:13:53,267 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:13:53,267 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:13:53,268 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:13:53,268 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:13:53,268 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:13:53,269 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:13:53,269 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:13:53,270 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:13:53,271 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:13:53,308 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:13:53,308 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:13:53,308 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:13:53,309 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:13:53,309 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:13:53,309 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:13:53,309 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:13:53,310 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:13:53,313 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:13:53,314 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:13:53,314 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:13:53,314 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:13:53,615 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:13:57,315 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:02,316 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:03,756 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:14:03,756 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:14:03,756 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:14:03,757 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:14:03,757 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:14:03,758 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:14:03,758 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:14:03,759 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:14:03,761 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:14:04,297 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:14:04,297 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:14:04,297 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:14:04,297 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:14:04,298 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:14:04,298 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:14:04,298 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:14:04,298 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:14:04,305 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:14:04,305 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:14:04,305 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:14:04,306 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:14:06,620 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:14:07,301 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:14:07,302 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:14:07,302 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:14:07,438 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:12,439 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:17,441 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:22,301 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:14:22,301 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:14:22,341 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:14:22,452 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:27,453 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:32,454 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:34,346 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:14:34,346 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:14:34,346 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:14:34,346 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:14:34,347 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:14:34,347 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:14:34,348 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:14:34,350 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:14:34,350 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:14:36,179 DEBUG SenderThread:98 [sender.py:send():379] send: config +2024-04-10 16:14:36,181 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:14:36,181 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:14:36,181 WARNING SenderThread:98 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-10 16:14:36,631 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:14:37,301 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:14:37,301 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:14:37,304 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:14:38,443 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:43,445 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:48,445 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:52,301 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:14:52,301 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:14:52,341 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:14:54,428 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:14:59,429 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:04,435 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:04,641 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/config.yaml +2024-04-10 16:15:07,302 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:15:07,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:15:07,303 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:15:10,428 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:15,429 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:20,430 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:22,302 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:15:22,302 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:15:22,342 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:15:26,420 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:28,713 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: partial_history +2024-04-10 16:15:28,714 DEBUG SenderThread:98 [sender.py:send():379] send: history +2024-04-10 16:15:28,714 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: summary_record +2024-04-10 16:15:28,715 INFO SenderThread:98 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-10 16:15:29,650 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/wandb-summary.json +2024-04-10 16:15:31,715 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:34,096 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:15:34,096 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:15:34,096 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:15:34,097 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:15:34,097 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:15:34,097 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:15:34,098 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:15:34,099 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:15:34,104 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:15:34,104 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:15:34,104 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:15:34,105 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:15:37,105 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:37,302 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:15:37,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:15:37,303 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:15:43,015 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:48,016 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:52,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:15:52,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:15:52,303 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:15:53,435 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:15:58,437 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:03,438 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:07,302 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:16:07,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:16:07,304 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:16:08,529 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:13,530 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:18,531 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:22,302 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:16:22,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:16:22,304 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:16:24,369 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:29,370 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:34,371 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:34,711 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:16:34,711 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:16:34,711 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:16:34,712 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:16:34,712 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:16:34,713 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:16:34,715 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:16:34,716 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:16:34,716 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:16:34,751 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:16:34,751 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:16:34,751 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:16:34,752 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:16:34,752 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:16:34,752 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:16:34,752 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:16:34,753 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:16:34,757 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:16:34,757 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:16:34,757 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:16:34,758 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:16:37,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:16:37,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:16:37,304 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:16:39,471 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:44,472 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:49,473 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:52,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:16:52,303 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:16:52,343 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:16:55,381 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:16:59,060 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:16:59,060 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:16:59,061 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:16:59,061 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:16:59,061 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:16:59,062 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:16:59,062 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:16:59,063 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:16:59,064 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:16:59,101 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:16:59,101 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:16:59,101 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:16:59,102 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:16:59,102 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:16:59,102 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:16:59,102 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:16:59,103 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:16:59,106 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:16:59,107 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:16:59,107 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:16:59,107 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:01,108 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:05,778 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:05,778 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:05,778 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:05,778 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:05,779 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:05,780 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:05,780 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:05,781 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:05,781 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:05,829 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:05,829 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:05,829 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:05,829 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:05,829 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:05,829 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:05,830 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:05,830 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:05,835 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:05,835 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:17:05,835 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:17:05,836 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:06,836 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:07,303 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:17:07,304 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:17:07,304 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:17:07,955 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:07,955 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:07,955 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:07,956 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:07,956 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:07,957 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:07,958 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:07,960 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:07,961 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:07,995 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:07,995 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:07,995 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:07,996 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:07,996 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:07,996 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:07,996 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:07,997 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:08,001 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:08,001 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:17:08,001 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:17:08,002 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:09,862 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:09,862 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:09,863 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:09,863 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:09,863 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:09,864 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:09,865 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:09,867 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:09,868 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:09,869 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:09,869 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:09,869 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:09,870 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:09,870 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:09,870 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:09,871 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:09,873 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:09,879 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:09,880 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:17:09,880 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:17:09,880 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:11,881 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:12,690 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:17:12,728 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:12,728 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:12,728 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:12,729 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:12,729 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:12,730 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:12,730 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:12,730 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:12,731 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:12,739 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:12,739 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:12,739 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:12,739 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:12,740 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:12,740 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:12,741 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:12,741 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:12,746 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:12,747 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:17:12,747 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:17:12,747 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:13,821 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:13,821 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:13,821 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:13,821 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:13,822 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:13,822 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:13,823 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:13,824 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:13,824 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:13,831 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:13,831 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:13,832 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:13,832 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:13,832 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:13,832 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:13,832 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:13,835 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:13,839 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:13,839 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:17:13,839 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:17:13,840 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:14,812 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:14,813 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:14,813 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:14,813 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:14,813 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:14,814 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:14,815 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:14,815 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:14,816 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:15,660 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:15,660 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:15,660 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:15,660 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:15,660 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:15,661 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:15,661 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:15,661 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:15,667 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:15,667 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:17:15,667 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:17:15,667 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:16,692 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:17:17,668 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:17,682 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:17,682 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:17,682 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:17,682 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:17,683 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:17,683 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:17,684 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:17,684 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:17,685 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:17,692 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:17:17,705 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:17,705 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:17,705 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:17,705 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:17,705 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:17,705 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:17,705 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:17,706 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:17,711 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:17,711 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:17:17,712 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:17:17,712 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:18,381 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:18,381 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:18,381 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:18,382 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:18,382 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:18,383 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:18,384 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:18,387 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:18,389 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:18,393 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:18,393 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:18,393 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:18,393 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:18,394 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:18,394 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:18,394 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:18,394 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:18,399 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:18,399 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:17:18,400 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:17:18,400 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:19,695 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:19,695 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:19,695 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:19,696 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:19,696 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:19,697 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:19,697 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:19,698 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:19,699 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:19,718 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:19,718 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:19,718 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:19,718 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:19,718 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:19,719 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:19,721 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:19,721 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:19,725 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:19,725 INFO HandlerThread:98 [interfaces.py:finish():202] Joined memory monitor +2024-04-10 16:17:19,726 INFO HandlerThread:98 [interfaces.py:finish():202] Joined network monitor +2024-04-10 16:17:19,726 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:20,602 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:20,602 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:20,602 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:20,602 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:20,602 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:20,603 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:20,606 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: pause +2024-04-10 16:17:20,607 INFO HandlerThread:98 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-04-10 16:17:20,607 INFO HandlerThread:98 [system_monitor.py:finish():203] Stopping system monitor +2024-04-10 16:17:20,608 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:20,608 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:17:20,608 DEBUG SystemMonitor:98 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-04-10 16:17:20,608 DEBUG SystemMonitor:98 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-04-10 16:17:20,609 INFO HandlerThread:98 [interfaces.py:finish():202] Joined cpu monitor +2024-04-10 16:17:20,610 INFO HandlerThread:98 [interfaces.py:finish():202] Joined disk monitor +2024-04-10 16:17:20,622 INFO HandlerThread:98 [interfaces.py:finish():202] Joined gpu monitor +2024-04-10 16:17:20,622 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:17:21,871 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: resume +2024-04-10 16:17:21,871 INFO HandlerThread:98 [handler.py:handle_request_resume():699] starting system metrics thread +2024-04-10 16:17:21,871 INFO HandlerThread:98 [system_monitor.py:start():194] Starting system monitor +2024-04-10 16:17:21,871 INFO SystemMonitor:98 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-04-10 16:17:21,872 INFO SystemMonitor:98 [interfaces.py:start():190] Started cpu monitoring +2024-04-10 16:17:21,872 INFO SystemMonitor:98 [interfaces.py:start():190] Started disk monitoring +2024-04-10 16:17:21,874 INFO SystemMonitor:98 [interfaces.py:start():190] Started gpu monitoring +2024-04-10 16:17:21,875 INFO SystemMonitor:98 [interfaces.py:start():190] Started memory monitoring +2024-04-10 16:17:21,876 INFO SystemMonitor:98 [interfaces.py:start():190] Started network monitoring +2024-04-10 16:17:22,304 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:17:22,305 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:17:22,305 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:17:22,694 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:17:23,046 DEBUG SenderThread:98 [sender.py:send():379] send: config +2024-04-10 16:17:23,048 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:23,086 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:17:23,086 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:17:23,086 WARNING SenderThread:98 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-04-10 16:17:23,695 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:17:28,087 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:33,088 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:37,304 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:17:37,304 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:17:37,307 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:17:38,449 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:43,455 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:43,703 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/config.yaml +2024-04-10 16:17:48,597 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:52,304 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:17:52,305 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:17:52,305 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:17:54,374 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:17:59,375 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:18:04,376 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:18:07,305 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:18:07,305 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:18:07,305 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:18:09,443 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:18:14,444 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:18:15,566 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: partial_history +2024-04-10 16:18:15,567 DEBUG SenderThread:98 [sender.py:send():379] send: history +2024-04-10 16:18:15,567 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: summary_record +2024-04-10 16:18:15,568 INFO SenderThread:98 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-10 16:18:15,715 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/wandb-summary.json +2024-04-10 16:18:19,569 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:18:20,963 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: partial_history +2024-04-10 16:18:20,967 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:18:20,967 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:18:20,967 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:18:20,967 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:18:20,967 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:18:20,967 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:18:20,968 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:18:20,968 DEBUG SenderThread:98 [sender.py:send():379] send: metric +2024-04-10 16:18:20,974 DEBUG SenderThread:98 [sender.py:send():379] send: history +2024-04-10 16:18:20,974 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: summary_record +2024-04-10 16:18:20,975 INFO SenderThread:98 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-04-10 16:18:21,718 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/wandb-summary.json +2024-04-10 16:18:21,876 DEBUG SystemMonitor:98 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-04-10 16:18:21,877 DEBUG SenderThread:98 [sender.py:send():379] send: stats +2024-04-10 16:18:22,869 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:18:22,958 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:18:22,958 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:18:23,719 INFO Thread-12 :98 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_155650-albqrbhg/files/output.log +2024-04-10 16:18:25,087 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:18:30,088 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:18:35,089 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report +2024-04-10 16:18:37,440 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: internal_messages +2024-04-10 16:18:37,716 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: stop_status +2024-04-10 16:18:37,716 DEBUG SenderThread:98 [sender.py:send_request():406] send_request: stop_status +2024-04-10 16:18:40,900 DEBUG HandlerThread:98 [handler.py:handle_request():146] handle_request: status_report