diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/debug-internal.log" @@ -0,0 +1,2233 @@ +2024-05-24 20:27:37,422 INFO StreamThr :1201 [internal.py:wandb_internal():86] W&B internal server running at pid: 1201, started at: 2024-05-24 20:27:37.421983 +2024-05-24 20:27:37,424 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status +2024-05-24 20:27:38,192 INFO WriterThread:1201 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/run-n1w0kmmv.wandb +2024-05-24 20:27:38,192 DEBUG SenderThread:1201 [sender.py:send():379] send: header +2024-05-24 20:27:38,196 DEBUG SenderThread:1201 [sender.py:send():379] send: run +2024-05-24 20:27:38,444 INFO SenderThread:1201 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files +2024-05-24 20:27:38,445 INFO SenderThread:1201 [sender.py:_start_run_threads():1124] run started: n1w0kmmv with start time 1716582457.423424 +2024-05-24 20:27:38,448 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: check_version +2024-05-24 20:27:38,449 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: check_version +2024-05-24 20:27:38,519 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: run_start +2024-05-24 20:27:38,531 DEBUG HandlerThread:1201 [system_info.py:__init__():26] System info init +2024-05-24 20:27:38,531 DEBUG HandlerThread:1201 [system_info.py:__init__():41] System info init done +2024-05-24 20:27:38,531 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:27:38,531 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:27:38,531 INFO HandlerThread:1201 [system_monitor.py:probe():214] Collecting system info +2024-05-24 20:27:38,532 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:27:38,532 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:27:38,533 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:27:38,535 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:27:38,535 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:27:38,540 DEBUG HandlerThread:1201 [system_info.py:probe():150] Probing system +2024-05-24 20:27:38,543 DEBUG HandlerThread:1201 [gitlib.py:_init_repo():56] git repository is invalid +2024-05-24 20:27:38,543 DEBUG HandlerThread:1201 [system_info.py:probe():198] Probing system done +2024-05-24 20:27:38,543 DEBUG HandlerThread:1201 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-05-24T20:27:38.540904', 'startedAt': '2024-05-24T20:27:37.415317', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '890e8b3ca76b', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.17, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.17, 'min': 0.0, 'max': 0.0}, {'current': 2000.17, 'min': 0.0, 'max': 0.0}, {'current': 2000.17, 'min': 0.0, 'max': 0.0}, {'current': 2000.17, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5599.177722930908}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357559204101562}} +2024-05-24 20:27:38,543 INFO HandlerThread:1201 [system_monitor.py:probe():224] Finished collecting system info +2024-05-24 20:27:38,543 INFO HandlerThread:1201 [system_monitor.py:probe():227] Publishing system info +2024-05-24 20:27:38,544 DEBUG HandlerThread:1201 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-24 20:27:39,447 INFO Thread-12 :1201 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/conda-environment.yaml +2024-05-24 20:27:53,563 ERROR HandlerThread:1201 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +Traceback (most recent call last): + File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda + subprocess.call( + File "/opt/conda/lib/python3.10/subprocess.py", line 347, in call + return p.wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1209, in wait + return self._wait(timeout=timeout) + File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait + raise TimeoutExpired(self.args, timeout) +subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-05-24 20:27:53,567 DEBUG HandlerThread:1201 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-24 20:27:53,567 INFO HandlerThread:1201 [system_monitor.py:probe():229] Finished publishing system info +2024-05-24 20:27:53,583 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:27:53,583 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: keepalive +2024-05-24 20:27:53,584 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:27:53,584 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: keepalive +2024-05-24 20:27:53,584 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:27:53,584 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: keepalive +2024-05-24 20:27:53,584 DEBUG SenderThread:1201 [sender.py:send():379] send: files +2024-05-24 20:27:53,585 INFO SenderThread:1201 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-05-24 20:27:53,909 INFO wandb-upload_0:1201 [upload_job.py:push():131] Uploaded file /tmp/tmp15yhem4uwandb/afmhiuoa-wandb-metadata.json +2024-05-24 20:27:54,450 INFO Thread-12 :1201 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/wandb-metadata.json +2024-05-24 20:27:54,626 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: python_packages +2024-05-24 20:27:54,627 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: python_packages +2024-05-24 20:27:54,630 DEBUG SenderThread:1201 [sender.py:send():379] send: telemetry +2024-05-24 20:27:54,644 DEBUG SenderThread:1201 [sender.py:send():379] send: config +2024-05-24 20:27:54,646 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:27:54,646 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:27:54,647 DEBUG SenderThread:1201 [sender.py:send():379] send: telemetry +2024-05-24 20:27:54,647 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:27:54,647 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:27:54,648 WARNING SenderThread:1201 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-05-24 20:27:54,648 DEBUG SenderThread:1201 [sender.py:send():379] send: telemetry +2024-05-24 20:27:54,649 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:27:55,450 INFO Thread-12 :1201 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/requirements.txt +2024-05-24 20:27:57,799 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:27:59,452 INFO Thread-12 :1201 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:28:01,453 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:28:03,099 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:03,453 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:28:04,490 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:28:04,490 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:28:04,490 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:28:04,490 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:28:04,491 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:28:04,491 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:28:04,491 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:28:04,491 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:28:04,498 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:28:04,499 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:28:04,499 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:28:04,499 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:28:08,506 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:09,455 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/config.yaml +2024-05-24 20:28:09,627 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:28:09,628 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:28:09,628 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:28:13,755 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:18,756 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:23,758 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:24,627 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:28:24,628 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:28:24,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:28:28,774 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:33,775 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:38,776 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:39,627 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:28:39,628 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:28:39,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:28:44,755 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:49,756 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:54,628 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:28:54,628 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:28:54,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:28:55,750 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:28:58,368 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:28:58,369 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:28:58,369 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:28:58,369 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:28:58,370 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:28:58,371 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:28:58,371 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:28:58,372 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:28:58,373 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:28:59,626 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:28:59,626 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:28:59,626 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:28:59,627 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:28:59,627 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:28:59,627 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:28:59,628 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:28:59,628 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:28:59,634 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:28:59,634 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:28:59,634 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:28:59,634 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:29:01,476 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:29:01,635 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:06,637 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:08,950 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:29:08,950 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:29:08,950 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:29:08,951 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:29:08,952 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:29:08,952 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:29:08,953 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:29:08,953 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:29:08,953 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:29:08,953 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:29:08,953 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:29:08,954 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:29:08,954 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:29:08,956 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:29:08,957 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:29:09,628 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:29:09,629 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:29:09,631 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:29:11,764 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:12,142 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:29:12,142 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:29:12,142 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:29:12,142 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:29:12,143 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:29:12,144 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:29:12,145 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:29:12,146 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:29:12,147 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:29:16,764 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:16,857 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:29:16,857 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:29:16,857 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:29:16,858 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:29:16,858 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:29:16,858 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:29:16,859 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:29:16,858 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:29:16,865 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:29:16,866 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:29:16,866 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:29:16,866 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:29:20,765 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:29:20,765 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:29:20,766 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:29:20,766 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:29:20,766 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:29:20,767 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:29:20,768 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:29:20,768 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:29:20,769 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:29:21,867 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:24,479 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:29:24,479 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:29:24,479 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:29:24,479 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:29:24,479 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:29:24,479 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:29:24,480 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:29:24,480 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:29:24,486 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:29:24,487 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:29:24,487 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:29:24,487 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:29:24,628 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:29:24,629 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:29:24,629 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:29:27,772 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:30,222 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:29:30,222 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:29:30,222 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:29:30,222 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:29:30,223 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:29:30,224 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:29:30,225 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:29:30,225 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:29:30,227 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:29:32,773 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:37,774 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:39,989 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:29:39,991 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:29:39,991 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:29:43,111 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:48,112 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:53,113 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:29:54,986 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:29:54,987 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:29:55,027 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:29:58,139 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:03,141 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:08,142 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:11,417 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:30:11,515 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:30:11,515 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:30:13,683 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:18,683 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:23,684 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:25,234 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:30:25,235 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:30:25,275 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:30:29,325 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:30,227 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:30:30,229 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:30:35,230 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:40,235 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:30:40,235 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:30:40,275 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:30:40,378 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:45,073 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:30:45,073 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:30:45,073 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:30:45,073 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:30:45,074 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:30:45,075 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:30:45,075 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:30:45,082 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:30:45,082 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:30:45,082 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:30:45,083 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:30:46,083 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:51,084 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:55,235 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:30:55,235 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:30:55,275 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:30:56,391 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:30:58,708 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:30:58,708 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:30:58,708 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:30:58,708 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:30:58,709 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:30:58,709 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:30:58,712 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:30:58,712 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:30:58,713 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:30:58,748 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:30:58,748 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:30:58,748 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:30:58,749 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:30:58,749 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:30:58,749 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:30:58,750 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:30:58,750 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:30:58,756 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:30:58,756 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:30:58,756 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:30:58,757 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:31:01,758 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:02,189 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:31:02,189 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:31:02,189 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:31:02,190 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:31:02,190 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:31:02,191 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:31:02,192 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:31:02,193 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:31:02,194 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:31:02,614 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:31:02,614 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:31:02,614 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:31:02,614 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:31:02,614 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:31:02,615 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:31:02,616 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:31:02,616 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:31:02,622 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:31:02,622 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:31:02,622 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:31:02,622 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:31:04,726 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:31:04,726 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:31:04,726 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:31:04,726 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:31:04,727 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:31:04,728 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:31:04,728 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:31:04,728 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:31:04,729 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:31:04,730 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:31:04,730 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:31:04,731 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:31:04,732 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:31:04,732 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:31:04,733 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:31:04,734 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:31:04,740 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:31:04,740 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:31:04,741 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:31:05,525 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:31:07,065 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:31:07,065 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:31:07,066 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:31:07,066 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:31:07,066 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:31:07,067 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:31:07,069 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:31:07,069 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:31:07,070 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:31:07,102 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:07,299 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:31:07,299 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:31:07,299 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:31:07,300 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:31:07,300 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:31:07,300 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:31:07,301 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:31:07,301 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:31:07,309 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:31:07,309 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:31:07,309 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:31:07,309 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:31:09,527 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:31:10,188 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:31:10,188 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:31:10,188 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:31:10,189 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:31:10,190 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:31:10,190 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:31:10,191 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:31:10,192 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:31:10,196 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:31:10,228 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:31:10,229 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:31:10,229 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:31:10,229 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:31:10,229 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:31:10,229 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:31:10,230 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:31:10,230 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:31:10,235 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:31:10,236 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:31:10,236 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:31:10,236 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:31:10,237 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:31:10,237 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:31:10,237 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:31:11,527 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:31:12,348 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:12,405 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:31:12,405 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:31:12,405 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:31:12,405 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:31:12,406 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:31:12,407 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:31:12,408 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:31:12,409 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:31:12,409 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:31:17,349 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:22,350 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:26,690 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:31:26,690 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:31:26,809 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:31:27,767 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:32,768 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:37,769 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:41,096 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:31:41,097 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:31:41,263 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:31:43,503 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:48,504 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:50,148 DEBUG SenderThread:1201 [sender.py:send():379] send: config +2024-05-24 20:31:50,189 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:31:50,189 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:31:50,189 WARNING SenderThread:1201 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-05-24 20:31:54,188 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:31:55,543 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:31:56,551 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:31:56,552 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:31:56,666 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:31:58,627 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:31:58,627 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:31:58,627 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:31:58,627 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:31:58,628 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:31:58,628 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:31:58,628 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:31:58,628 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:31:58,635 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:31:58,635 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:31:58,635 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:31:58,635 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:31:59,545 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:31:59,636 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:04,637 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:09,638 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:11,450 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:32:11,450 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:32:11,451 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:32:15,595 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:20,601 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:21,553 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/config.yaml +2024-05-24 20:32:25,735 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:26,450 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:32:26,451 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:32:26,452 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:32:31,544 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:36,545 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:41,450 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:32:41,451 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:32:41,451 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:32:42,539 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:47,540 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:52,541 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:32:56,450 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:32:56,451 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:32:56,452 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:32:58,539 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:03,540 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:08,541 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:11,451 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:33:11,451 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:33:11,452 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:33:13,556 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:18,557 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:23,558 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:26,451 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:33:26,451 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:33:26,452 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:33:28,600 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:33,605 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:38,605 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:41,451 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:33:41,452 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:33:41,452 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:33:44,580 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:49,581 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:54,582 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:33:56,451 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:33:56,452 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:33:56,452 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:33:59,635 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:04,635 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:06,571 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:34:06,571 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:34:06,571 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:34:06,572 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:34:06,572 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:34:06,573 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:34:06,574 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:34:06,574 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:34:06,575 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:34:06,718 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:34:06,718 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:34:06,718 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:34:06,718 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:34:06,719 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:34:06,719 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:34:06,719 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:34:06,719 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:34:06,725 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:34:06,725 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:34:06,725 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:34:06,726 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:34:07,589 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:34:09,727 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:10,002 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:34:10,002 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:34:10,002 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:34:10,002 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:34:10,003 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:34:10,004 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:34:10,005 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:34:10,006 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:34:10,006 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:34:12,036 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:34:12,130 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:34:12,130 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:34:15,205 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:20,206 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:25,207 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:27,167 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:34:27,168 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:34:27,544 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:34:30,755 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:35,756 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:40,757 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:42,573 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:34:42,574 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:34:42,833 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:34:46,473 DEBUG SenderThread:1201 [sender.py:send():379] send: config +2024-05-24 20:34:46,475 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:46,509 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:34:46,509 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:34:46,509 WARNING SenderThread:1201 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-05-24 20:34:51,510 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:53,606 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:34:55,607 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:34:55,929 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:34:55,929 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:34:55,929 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:34:55,930 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:34:55,930 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:34:55,931 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:34:55,931 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:34:55,931 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:34:55,937 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:34:55,937 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:34:55,937 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:34:55,938 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:34:56,564 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:34:56,564 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:34:56,698 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:34:57,048 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:34:57,608 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/config.yaml +2024-05-24 20:35:01,845 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:06,846 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:11,564 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:35:11,564 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:35:12,049 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:35:12,679 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:17,680 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:22,681 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:26,131 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:35:26,131 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:35:26,131 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:35:26,132 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:35:26,132 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:35:26,133 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:35:26,133 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:35:26,134 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:35:26,136 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:35:26,172 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:35:26,172 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:35:26,172 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:35:26,173 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:35:26,173 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:35:26,174 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:35:26,174 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:35:26,174 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:35:26,179 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:35:26,180 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:35:26,180 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:35:26,180 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:35:26,564 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:35:26,564 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:35:27,049 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:35:28,619 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:35:28,680 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:32,729 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:35:32,729 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:35:32,729 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:35:32,730 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:35:32,730 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:35:32,732 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:35:32,733 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:35:32,734 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:35:32,737 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:35:32,770 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:35:32,770 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:35:32,770 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:35:32,770 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:35:32,771 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:35:32,771 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:35:32,771 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:35:32,772 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:35:32,776 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:35:32,776 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:35:32,777 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:35:32,777 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:35:33,778 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:34,730 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:35:34,730 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:35:34,730 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:35:34,730 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:35:34,731 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:35:34,732 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:35:34,733 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:35:34,733 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:35:34,733 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:35:34,734 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:35:34,734 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:35:34,734 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:35:34,734 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:35:34,736 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:35:34,737 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:35:34,746 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:35:34,747 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:35:36,622 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:35:39,748 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:41,564 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:35:41,565 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:35:42,049 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:35:45,680 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:47,484 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:35:47,484 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:35:47,484 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:35:47,484 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:35:47,485 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:35:47,486 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:35:47,486 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:35:47,487 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:35:47,488 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:35:47,524 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:35:47,524 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:35:47,524 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:35:47,525 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:35:47,525 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:35:47,525 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:35:47,525 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:35:47,525 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:35:47,530 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:35:47,531 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:35:47,531 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:35:47,531 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:35:51,532 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:56,249 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:35:56,250 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:35:56,250 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:35:56,250 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:35:56,250 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:35:56,252 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:35:56,253 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:35:56,254 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:35:56,254 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:35:56,254 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:35:56,255 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:35:56,256 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:35:56,256 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:35:56,256 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:35:56,257 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:35:56,257 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:35:56,264 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:35:56,264 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:35:56,264 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:35:56,564 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:35:56,565 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:35:56,696 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:35:57,049 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:36:01,697 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:06,698 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:11,565 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:36:11,565 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:36:12,049 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:36:12,680 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:17,681 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:22,682 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:26,565 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:36:26,565 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:36:27,049 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:36:27,709 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:32,710 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:37,711 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:41,565 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:36:41,566 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:36:42,050 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:36:43,700 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:48,701 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:53,702 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:36:56,565 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:36:56,566 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:36:57,050 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:36:59,681 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:04,682 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:09,683 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:11,566 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:37:11,566 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:37:12,050 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:37:15,681 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:20,367 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:37:20,367 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:37:20,367 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:37:20,367 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:37:20,368 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:37:20,368 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:37:20,369 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:37:20,371 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:37:20,374 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:37:20,407 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:37:20,407 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:37:20,407 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:37:20,408 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:37:20,408 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:37:20,408 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:37:20,408 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:37:20,409 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:37:20,414 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:37:20,414 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:37:20,414 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:37:20,414 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:37:21,415 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:26,416 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:26,566 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:37:26,566 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:37:27,050 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:37:28,481 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:37:28,482 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:37:28,482 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:37:28,482 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:37:28,482 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:37:28,483 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:37:28,484 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:37:28,485 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:37:28,485 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:37:28,522 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:37:28,522 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:37:28,522 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:37:28,523 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:37:28,523 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:37:28,523 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:37:28,523 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:37:28,524 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:37:28,528 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:37:28,529 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:37:28,529 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:37:28,529 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:37:31,530 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:36,531 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:41,532 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:41,566 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:37:41,566 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:37:42,050 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:37:46,681 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:51,682 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:37:56,566 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:37:56,566 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:37:57,051 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:37:57,681 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:00,947 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:38:00,947 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:38:00,947 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:38:00,947 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:38:00,948 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:38:00,948 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:38:00,949 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:38:00,951 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:38:00,954 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:38:00,987 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:38:00,987 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:38:00,987 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:38:00,988 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:38:00,988 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:38:00,988 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:38:00,988 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:38:00,988 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:38:00,993 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:38:00,994 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:38:00,994 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:38:00,994 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:38:02,995 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:04,525 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:38:04,525 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:38:04,525 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:38:04,525 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:38:04,526 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:38:04,527 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:38:04,528 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:38:04,529 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:38:04,529 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:38:04,550 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:38:04,550 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:38:04,550 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:38:04,551 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:38:04,551 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:38:04,551 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:38:04,551 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:38:04,551 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:38:04,558 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:38:04,558 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:38:04,558 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:38:04,558 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:38:08,559 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:11,566 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:38:11,567 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:38:12,051 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:38:12,369 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:38:12,369 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:38:12,369 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:38:12,370 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:38:12,370 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:38:12,371 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:38:12,372 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:38:12,373 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:38:12,373 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:38:12,433 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:38:12,433 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:38:12,433 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:38:12,433 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:38:12,434 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:38:12,434 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:38:12,434 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:38:12,435 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:38:12,440 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:38:12,440 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:38:12,441 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:38:12,441 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:38:14,442 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:14,677 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:38:16,939 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:38:16,939 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:38:16,939 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:38:16,939 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:38:16,940 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:38:16,941 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:38:16,941 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:38:16,943 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:38:16,944 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:38:16,999 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:38:16,999 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:38:16,999 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:38:16,999 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:38:16,999 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:38:17,000 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:38:17,000 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:38:17,000 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:38:17,006 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:38:17,007 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:38:17,007 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:38:17,007 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:38:20,008 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:21,249 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:38:21,249 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:38:21,249 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:38:21,250 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:38:21,250 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:38:21,251 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:38:21,251 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:38:21,252 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:38:21,253 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:38:21,260 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:38:21,261 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:38:21,261 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:38:21,261 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:38:21,261 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:38:21,261 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:38:21,262 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:38:21,262 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:38:21,268 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:38:21,268 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:38:21,268 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:38:21,269 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:38:22,931 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:38:22,931 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:38:22,931 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:38:22,931 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:38:22,932 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:38:22,932 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:38:22,933 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:38:22,934 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:38:22,935 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:38:22,941 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:38:22,941 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:38:22,941 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:38:22,941 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:38:22,941 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:38:22,941 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:38:22,942 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:38:22,943 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:38:22,950 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:38:22,950 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:38:22,950 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:38:22,951 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:38:25,952 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:26,566 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:38:26,567 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:38:27,051 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:38:31,742 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:36,744 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:37,852 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:38:37,852 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:38:37,852 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:38:37,852 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:38:37,853 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:38:37,854 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:38:37,855 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:38:37,856 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:38:37,857 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:38:41,566 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:38:41,567 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:38:42,051 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:38:42,724 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:47,725 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:52,726 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:55,255 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:38:55,255 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:38:55,255 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:38:55,255 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:38:55,256 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:38:55,256 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:38:55,256 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:38:55,257 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:38:55,262 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:38:55,263 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:38:55,263 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:38:55,263 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:38:56,567 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:38:56,567 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:38:57,051 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:38:57,733 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:38:58,789 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:38:58,789 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:38:58,789 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:38:58,789 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:38:58,790 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:38:58,792 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:38:58,792 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:38:58,793 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:38:58,794 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:38:58,829 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:38:58,829 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:38:58,829 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:38:58,830 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:38:58,830 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:38:58,830 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:38:58,831 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:38:58,831 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:38:58,836 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:38:58,836 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:38:58,836 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:38:58,837 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:39:00,763 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:39:00,763 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:39:00,763 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:39:00,763 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:39:00,764 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:39:00,765 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:39:00,765 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:39:00,766 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:39:00,767 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:39:02,838 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:07,838 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:11,567 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:39:11,567 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:39:12,051 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:39:13,682 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:18,682 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:23,683 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:26,567 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:39:26,567 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:39:27,052 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:39:29,682 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:34,683 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:39,684 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:41,567 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:39:41,568 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:39:42,052 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:39:45,682 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:50,683 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:55,684 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:39:56,567 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:39:56,568 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:39:57,052 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:40:00,767 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:40:00,768 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:40:00,769 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:05,770 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:10,770 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:11,567 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:40:11,568 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:40:12,052 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:40:16,334 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:40:16,334 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:40:16,334 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:40:16,335 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:40:16,335 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:40:16,337 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:40:16,337 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:40:16,347 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:40:16,347 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:40:16,347 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:40:16,348 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:40:16,348 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:21,350 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:21,473 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:40:21,473 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:40:21,473 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:40:21,474 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:40:21,474 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:40:21,475 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:40:21,476 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:40:21,477 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:40:21,478 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:40:22,683 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:40:22,683 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:40:22,683 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:40:22,683 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:40:22,683 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:40:22,684 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:40:22,685 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:40:22,685 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:40:22,690 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:40:22,690 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:40:22,691 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:40:22,691 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:40:23,720 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:40:26,568 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:40:26,568 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:40:26,640 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:27,052 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:40:28,980 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:40:28,980 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:40:28,980 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:40:28,981 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:40:28,981 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:40:28,982 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:40:28,983 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:40:28,984 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:40:28,985 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:40:31,683 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:33,962 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:40:33,962 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:40:33,962 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:40:33,963 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:40:33,963 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:40:33,964 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:40:33,964 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:40:33,964 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:40:33,970 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:40:33,970 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:40:33,970 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:40:33,971 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:40:36,972 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:36,995 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:40:36,995 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:40:36,995 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:40:36,995 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:40:36,996 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:40:36,997 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:40:36,998 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:40:36,999 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:40:36,999 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:40:36,999 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:40:37,000 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:40:37,002 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:40:37,002 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:40:37,002 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:40:37,002 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:40:37,003 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:40:37,009 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:40:37,009 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:40:37,010 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:40:41,568 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:40:41,568 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:40:42,052 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:40:42,203 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:40:42,203 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:40:42,204 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:40:42,204 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:40:42,204 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:40:42,205 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:40:42,206 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:40:42,207 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:40:42,207 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:40:42,437 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:42,621 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:40:42,621 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:40:42,621 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:40:42,622 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:40:42,622 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:40:42,622 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:40:42,623 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:40:42,623 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:40:42,629 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:40:42,630 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:40:42,630 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:40:42,630 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:40:43,727 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:40:44,474 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:40:44,474 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:40:44,474 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:40:44,474 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:40:44,475 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:40:44,476 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:40:44,476 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:40:44,477 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:40:44,477 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:40:44,477 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:40:44,478 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:40:44,478 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:40:44,478 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:40:44,478 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:40:44,478 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:40:44,481 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:40:44,489 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:40:44,489 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:40:44,490 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:40:47,491 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:48,432 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:40:48,432 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:40:48,432 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:40:48,433 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:40:48,433 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:40:48,434 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:40:48,434 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:40:48,436 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:40:48,437 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:40:48,669 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:40:48,669 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:40:48,670 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:40:48,670 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:40:48,670 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:40:48,670 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:40:48,671 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:40:48,671 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:40:48,677 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:40:48,677 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:40:48,678 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:40:48,678 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:40:49,729 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:40:50,725 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:40:50,725 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:40:50,725 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:40:50,726 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:40:50,726 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:40:50,727 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:40:50,731 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:40:50,731 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:40:50,732 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:40:50,732 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:40:50,734 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:40:50,734 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:40:50,735 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:40:50,735 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:40:50,735 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:40:50,736 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:40:50,738 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:40:50,743 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:40:50,743 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:40:50,743 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:40:50,744 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:40:51,733 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:40:52,538 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:40:52,538 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:40:52,538 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:40:52,539 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:40:52,539 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:40:52,540 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:40:52,541 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:40:52,541 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:40:52,542 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:40:52,745 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:57,317 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:40:57,318 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:40:57,843 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:40:57,923 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:41:02,844 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:07,845 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:12,966 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:41:12,966 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:41:13,071 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:13,772 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:41:18,072 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:23,073 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:27,267 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:41:27,840 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:41:27,840 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:41:28,311 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:30,455 DEBUG SenderThread:1201 [sender.py:send():379] send: config +2024-05-24 20:41:30,495 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:41:30,495 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:41:30,495 WARNING SenderThread:1201 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-05-24 20:41:33,495 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:37,749 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:41:38,518 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:39,044 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:41:39,044 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:41:39,044 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:41:39,045 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:41:39,045 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:41:39,045 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:41:39,046 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:41:39,046 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:41:39,053 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:41:39,053 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:41:39,053 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:41:39,054 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:41:39,749 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:41:41,634 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:41:41,635 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:41:42,124 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:41:43,901 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:48,907 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:49,753 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/config.yaml +2024-05-24 20:41:54,047 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:41:56,634 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:41:56,635 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:41:57,124 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:41:59,901 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:04,902 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:09,903 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:11,635 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:42:11,635 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:42:12,124 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:42:15,902 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:20,903 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:25,904 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:26,635 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:42:26,635 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:42:26,951 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:42:26,951 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:42:26,951 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:42:26,951 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:42:26,952 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:42:26,953 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:42:26,954 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:42:26,955 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:42:26,956 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:42:27,124 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:42:27,129 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:42:27,129 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:42:27,129 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:42:27,130 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:42:27,130 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:42:27,130 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:42:27,130 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:42:27,130 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:42:27,136 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:42:27,137 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:42:27,137 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:42:27,141 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:42:27,767 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:42:29,872 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:42:29,872 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:42:29,872 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:42:29,872 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:42:29,873 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:42:29,874 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:42:29,875 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:42:29,876 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:42:29,877 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:42:31,142 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:36,143 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:41,144 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:42,759 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:42:42,760 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:42:42,848 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:42:46,201 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:51,202 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:56,203 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:42:58,439 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:42:58,525 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:42:58,526 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:43:01,682 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:06,683 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:07,282 DEBUG SenderThread:1201 [sender.py:send():379] send: config +2024-05-24 20:43:07,283 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:43:07,283 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:43:07,283 WARNING SenderThread:1201 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-05-24 20:43:11,666 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:43:11,666 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:43:11,766 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:11,783 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:43:12,248 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:43:12,248 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:43:12,248 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:43:12,248 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:43:12,248 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:43:12,248 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:43:12,249 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:43:12,250 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:43:12,256 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:43:12,256 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:43:12,256 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:43:12,257 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:43:12,280 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:43:17,258 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:22,265 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:22,787 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/config.yaml +2024-05-24 20:43:26,667 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:43:26,667 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:43:27,281 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:43:28,250 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:33,251 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:38,252 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:41,667 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:43:41,667 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:43:42,281 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:43:44,250 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:49,252 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:54,253 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:43:56,667 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:43:56,667 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:43:57,281 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:44:00,251 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:05,252 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:08,644 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:44:08,645 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:44:08,645 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:44:08,645 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:44:08,646 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:44:08,646 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:44:08,647 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:44:08,647 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:44:08,648 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:44:10,253 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:11,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:44:11,669 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:44:12,455 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:44:14,295 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:44:14,295 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:44:14,295 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:44:14,296 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:44:14,296 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:44:14,297 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:44:14,297 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:44:14,297 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:44:14,302 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:44:14,303 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:44:14,303 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:44:14,303 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:44:15,304 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:20,305 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:25,306 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:26,667 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:44:26,667 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:44:27,455 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:44:30,336 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:35,337 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:39,720 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:44:39,720 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:44:39,720 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:44:39,720 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:44:39,721 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:44:39,721 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:44:39,722 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:44:39,723 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:44:39,724 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:44:39,771 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:44:39,771 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:44:39,771 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:44:39,771 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:44:39,772 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:44:39,772 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:44:39,772 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:44:39,772 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:44:39,778 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:44:39,778 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:44:39,778 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:44:39,779 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:44:40,779 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:40,819 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:44:41,667 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:44:41,668 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:44:42,455 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:44:43,544 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:44:43,544 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:44:43,544 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:44:43,544 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:44:43,545 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:44:43,546 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:44:43,546 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:44:43,547 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:44:43,548 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:44:46,337 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:51,338 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:56,339 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:44:56,667 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:44:56,668 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:44:57,455 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:45:02,338 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:07,339 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:11,667 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:45:11,668 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:45:12,455 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:45:13,337 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:18,338 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:23,339 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:26,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:45:26,668 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:45:27,455 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:45:29,337 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:34,338 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:39,339 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:41,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:45:41,668 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:45:42,456 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:45:43,548 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:45:43,549 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:45:44,550 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:49,551 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:54,551 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:45:56,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:45:56,668 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:45:57,456 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:46:00,337 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:05,338 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:10,339 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:11,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:46:11,668 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:46:12,456 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:46:13,550 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:46:15,551 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:20,552 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:25,553 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:26,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:46:26,668 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:46:27,456 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:46:31,143 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:31,183 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:46:31,183 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:46:31,184 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:46:31,184 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:46:31,184 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:46:31,186 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:46:31,186 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:46:31,191 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:46:31,191 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:46:31,191 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:46:31,192 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:46:33,859 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:46:36,193 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:41,194 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:41,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:46:41,669 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:46:42,456 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:46:46,338 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:49,362 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:46:49,363 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:46:49,363 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:46:49,363 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:46:49,364 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:46:49,364 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:46:49,366 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:46:49,366 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:46:49,367 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:46:49,403 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:46:49,403 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:46:49,403 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:46:49,404 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:46:49,404 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:46:49,404 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:46:49,404 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:46:49,405 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:46:49,410 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:46:49,410 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:46:49,410 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:46:49,411 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:46:51,412 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:52,170 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:46:52,170 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:46:52,170 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:46:52,171 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:46:52,171 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:46:52,172 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:46:52,173 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:46:52,174 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:46:52,175 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:46:54,392 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:46:54,392 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:46:54,392 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:46:54,392 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:46:54,393 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:46:54,393 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:46:54,393 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:46:54,394 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:46:54,399 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:46:54,399 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:46:54,399 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:46:54,400 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:46:56,668 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:46:56,669 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:46:56,798 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:46:57,456 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:46:58,318 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:46:58,318 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:46:58,318 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:46:58,319 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:46:58,319 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:46:58,320 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:46:58,321 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:46:58,322 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:46:58,323 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:46:58,358 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:46:58,358 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:46:58,358 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:46:58,359 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:46:58,359 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:46:58,359 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:46:58,359 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:46:58,359 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:46:58,364 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:46:58,365 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:46:58,365 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:46:58,365 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:47:02,366 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:06,483 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:47:06,483 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:47:06,483 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:47:06,484 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:47:06,484 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:47:06,485 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:47:06,487 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:47:06,488 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:47:06,488 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:47:07,465 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:07,831 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:47:07,831 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:47:07,831 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:47:07,831 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:47:07,831 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:47:07,831 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:47:07,832 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:47:07,832 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:47:07,838 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:47:07,838 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:47:07,838 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:47:07,839 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:47:08,873 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:47:10,834 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:47:10,834 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:47:10,834 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:47:10,835 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:47:10,835 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:47:10,836 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:47:10,837 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:47:10,838 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:47:10,839 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:47:10,840 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:47:10,840 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:47:10,841 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:47:10,841 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:47:10,842 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:47:10,842 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:47:10,842 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:47:10,845 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:47:10,850 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:47:10,850 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:47:10,850 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:47:10,850 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:47:10,874 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:47:11,669 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:47:11,669 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:47:12,456 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:47:13,338 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:14,003 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:47:14,003 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:47:14,003 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:47:14,004 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:47:14,004 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:47:14,005 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:47:14,006 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:47:14,008 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:47:14,008 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:47:18,339 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:23,340 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:26,669 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:47:26,670 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:47:27,457 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:47:29,338 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:34,339 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:39,340 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:41,670 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:47:41,671 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:47:43,369 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:47:45,308 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:50,309 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:55,310 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:47:56,761 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:47:56,761 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:47:57,653 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:48:00,392 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:02,461 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:48:02,461 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:48:02,461 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:48:02,461 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:48:02,462 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:48:02,462 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:48:02,462 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:48:02,463 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:48:02,469 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:48:02,469 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:48:02,469 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:48:02,470 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:48:05,471 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:05,732 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:48:05,732 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:48:05,732 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:48:05,732 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:48:05,733 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:48:05,734 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:48:05,735 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:48:05,735 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:48:05,735 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:48:05,736 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:48:05,737 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:48:05,737 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:48:05,737 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:48:05,737 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:48:05,738 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:48:05,738 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:48:05,739 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:48:08,287 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:48:08,288 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:48:08,288 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:48:08,288 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:48:08,289 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:48:08,289 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:48:08,290 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:48:08,291 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:48:08,292 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:48:08,714 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:48:08,715 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:48:08,715 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:48:08,715 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:48:08,716 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:48:08,716 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:48:08,716 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:48:08,716 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:48:08,722 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:48:08,723 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:48:08,723 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:48:08,723 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:48:10,291 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:48:10,291 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:48:10,291 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:48:10,292 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:48:10,292 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:48:10,293 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:48:10,295 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:48:10,295 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:48:10,295 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:48:10,296 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:48:10,296 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:48:10,296 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:48:10,296 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:48:10,296 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:48:10,297 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:48:10,301 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:48:10,313 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:48:10,314 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:48:10,314 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:48:10,904 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:48:11,315 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:11,693 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:48:11,693 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:48:12,653 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:48:13,842 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:48:13,842 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:48:13,842 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:48:13,842 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:48:13,843 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:48:13,844 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:48:13,844 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:48:13,849 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:48:13,849 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:48:14,092 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:48:14,092 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:48:14,092 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:48:14,093 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:48:14,093 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:48:14,093 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:48:14,094 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:48:14,094 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:48:14,099 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:48:14,100 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:48:14,100 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:48:14,100 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:48:16,126 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:48:16,126 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:48:16,126 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:48:16,126 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:48:16,127 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:48:16,127 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:48:16,128 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:48:16,128 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:48:16,130 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:48:16,906 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:48:17,101 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:22,102 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:27,103 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:27,411 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:48:27,411 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:48:28,538 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:48:32,871 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:37,872 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:40,378 DEBUG SenderThread:1201 [sender.py:send():379] send: config +2024-05-24 20:48:40,419 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:48:40,419 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 20:48:40,419 WARNING SenderThread:1201 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-05-24 20:48:42,052 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:48:42,053 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:48:42,632 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:48:42,633 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:48:42,633 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:48:42,633 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:48:42,633 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:48:42,633 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:48:42,634 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:48:42,634 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:48:42,640 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:48:42,641 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:48:42,641 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:48:42,641 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:48:42,696 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:48:42,916 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:48:43,642 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:44,917 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 20:48:48,643 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:53,644 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:48:56,982 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:48:56,983 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:48:57,696 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:48:59,418 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:00,382 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:49:00,382 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:49:00,382 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:49:00,382 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:49:00,383 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:49:00,384 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:49:00,384 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:49:00,385 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:49:00,385 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:49:00,425 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:49:00,426 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:49:00,426 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:49:00,426 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:49:00,426 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:49:00,427 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:49:00,427 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:49:00,427 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:49:00,433 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:49:00,433 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:49:00,433 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:49:00,434 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:49:03,722 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:49:03,722 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:49:03,722 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:49:03,723 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:49:03,723 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:49:03,724 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:49:03,725 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:49:03,726 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:49:03,727 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:49:03,727 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:49:03,727 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:49:03,728 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:49:03,728 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:49:03,728 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:49:03,729 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:49:03,730 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:49:03,737 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:49:03,737 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:49:03,738 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:49:04,738 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:08,089 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:49:08,090 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:49:08,090 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:49:08,090 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:49:08,090 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:49:08,096 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:49:08,100 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:49:08,105 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:49:08,105 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:49:08,456 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:49:08,456 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:49:08,456 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:49:08,457 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:49:08,457 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:49:08,457 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:49:08,458 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:49:08,458 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:49:08,465 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:49:08,465 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:49:08,465 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:49:08,466 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:49:10,472 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:10,927 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/config.yaml +2024-05-24 20:49:11,982 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:49:11,983 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:49:12,696 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:49:16,418 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:21,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:26,420 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:26,982 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:49:26,983 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:49:27,697 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:49:32,418 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:37,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:41,983 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:49:41,983 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:49:42,697 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:49:43,418 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:48,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:53,420 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:49:56,983 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:49:56,983 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:49:57,697 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:49:59,418 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:04,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:09,420 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:11,983 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:50:11,984 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:50:12,697 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:50:15,418 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:20,420 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:25,421 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:26,983 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:50:26,984 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:50:27,697 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:50:31,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:36,420 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:41,421 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:41,983 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:50:41,984 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:50:42,697 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:50:47,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:52,420 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:50:56,983 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:50:56,984 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:50:57,698 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:50:58,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:03,420 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:08,421 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:11,984 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:51:11,984 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:51:12,698 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:51:14,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:19,420 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:24,421 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:26,984 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:51:26,984 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:51:27,698 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:51:30,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:35,421 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:40,422 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:41,984 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:51:41,984 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:51:42,698 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:51:46,420 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:51,421 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:52,780 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:51:52,780 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:51:52,780 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:51:52,780 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:51:52,781 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:51:52,783 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:51:52,783 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:51:52,784 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:51:52,785 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:51:52,820 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:51:52,820 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:51:52,820 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:51:52,821 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:51:52,821 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:51:52,821 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:51:52,822 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:51:52,822 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:51:52,827 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:51:52,827 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:51:52,827 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:51:52,828 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:51:56,595 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:51:56,595 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:51:56,595 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:51:56,595 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:51:56,596 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:51:56,596 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:51:56,597 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:51:56,598 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:51:56,598 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:51:56,829 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:51:57,579 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:51:57,580 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:51:58,096 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:52:02,727 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:07,728 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:12,728 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:13,058 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:52:13,846 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:52:13,846 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:52:18,008 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:22,760 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:52:22,760 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:52:22,760 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:52:22,760 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:52:22,761 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:52:22,761 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:52:22,761 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:52:22,762 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:52:22,768 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:52:22,768 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:52:22,768 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:52:22,768 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:52:23,769 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:27,592 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:52:27,593 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:52:27,801 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:52:29,696 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:34,697 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:39,698 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:42,592 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:52:42,593 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:52:42,801 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:52:45,697 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:49,571 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:52:49,571 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:52:49,571 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:52:49,572 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:52:49,572 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:52:49,573 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:52:49,574 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:52:49,576 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:52:49,577 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:52:49,612 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: pause +2024-05-24 20:52:49,612 INFO HandlerThread:1201 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-24 20:52:49,612 INFO HandlerThread:1201 [system_monitor.py:finish():203] Stopping system monitor +2024-05-24 20:52:49,613 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:52:49,613 DEBUG SystemMonitor:1201 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-24 20:52:49,613 DEBUG SystemMonitor:1201 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-24 20:52:49,613 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined cpu monitor +2024-05-24 20:52:49,614 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined disk monitor +2024-05-24 20:52:49,619 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined gpu monitor +2024-05-24 20:52:49,619 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined memory monitor +2024-05-24 20:52:49,619 INFO HandlerThread:1201 [interfaces.py:finish():202] Joined network monitor +2024-05-24 20:52:49,620 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:52:51,620 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:52,108 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: resume +2024-05-24 20:52:52,108 INFO HandlerThread:1201 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-24 20:52:52,108 INFO HandlerThread:1201 [system_monitor.py:start():194] Starting system monitor +2024-05-24 20:52:52,108 INFO SystemMonitor:1201 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-24 20:52:52,109 INFO SystemMonitor:1201 [interfaces.py:start():190] Started cpu monitoring +2024-05-24 20:52:52,110 INFO SystemMonitor:1201 [interfaces.py:start():190] Started disk monitoring +2024-05-24 20:52:52,111 INFO SystemMonitor:1201 [interfaces.py:start():190] Started gpu monitoring +2024-05-24 20:52:52,112 INFO SystemMonitor:1201 [interfaces.py:start():190] Started memory monitoring +2024-05-24 20:52:52,112 INFO SystemMonitor:1201 [interfaces.py:start():190] Started network monitoring +2024-05-24 20:52:56,621 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:52:58,183 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:52:58,332 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:52:58,333 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:53:02,466 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:07,467 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:12,468 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:13,432 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:53:13,433 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:53:13,457 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:53:17,514 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:22,515 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:27,516 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:27,789 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:53:27,790 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:53:27,992 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:53:32,860 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:37,861 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:42,754 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:53:42,755 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:53:42,992 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:53:43,860 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:48,861 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:52,113 DEBUG SystemMonitor:1201 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-24 20:53:52,114 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:53:54,115 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:53:57,755 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:53:57,755 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:53:57,992 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:53:59,860 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:04,861 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:09,862 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:13,008 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:54:13,009 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:54:13,049 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:54:15,178 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:20,178 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:22,115 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:54:26,116 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:27,943 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:54:27,944 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:54:27,997 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:54:32,099 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:37,100 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:42,100 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:42,943 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:54:42,944 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:54:42,997 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:54:47,118 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:52,116 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:54:53,117 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:54:57,943 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:54:57,944 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:54:57,997 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:54:59,106 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:04,107 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:09,107 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:12,944 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:55:12,944 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:55:12,997 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:55:15,093 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:20,094 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:22,116 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:55:25,117 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:27,944 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:55:27,944 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:55:27,997 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:55:31,033 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:36,034 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:41,035 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:42,944 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:55:42,944 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:55:42,998 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:55:47,032 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:52,033 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:52,117 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:55:57,118 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:55:58,106 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:55:58,106 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:55:58,149 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:56:02,221 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:07,222 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:12,223 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:13,071 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:56:13,071 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:56:13,076 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:56:17,246 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:22,118 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:56:23,119 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:28,071 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:56:28,071 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:56:28,076 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:56:28,162 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:33,163 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:38,164 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:43,071 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:56:43,072 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:56:43,076 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:56:44,145 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:49,146 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:52,119 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:56:55,120 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:56:58,271 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:56:58,272 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:56:58,272 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:57:00,417 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:05,418 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:10,419 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:13,238 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:57:13,238 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:57:13,278 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:57:16,351 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:21,353 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:22,120 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:57:27,121 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:28,493 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:57:28,494 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:57:28,494 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:57:32,605 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:37,606 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:42,607 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:43,809 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:57:43,810 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:57:43,811 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:57:47,943 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:52,120 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:57:53,121 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:58,122 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:57:58,979 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:57:58,979 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:57:58,980 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:58:04,059 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:09,060 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:14,061 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:14,520 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:58:14,531 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:58:14,531 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:58:19,644 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:22,121 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:58:25,122 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:29,444 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:58:29,444 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:58:29,445 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:58:30,620 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:35,621 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:40,622 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:44,444 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:58:44,444 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:58:44,445 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:58:46,614 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:51,615 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:52,122 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:58:57,123 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:58:59,444 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:58:59,444 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:58:59,445 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:59:02,518 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:07,519 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:12,520 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:14,444 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:59:14,444 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:59:14,445 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:59:17,605 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:22,123 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:59:23,124 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:28,125 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:29,710 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:59:29,711 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:59:29,712 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:59:33,891 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:38,893 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:43,894 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:44,696 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:59:44,698 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:59:44,698 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 20:59:49,786 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:52,124 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 20:59:55,125 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 20:59:59,669 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 20:59:59,680 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 20:59:59,680 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:00:00,779 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:05,780 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:10,780 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:15,217 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:00:15,218 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:00:15,258 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:00:16,355 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:21,356 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:22,125 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 21:00:27,126 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:30,125 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:00:30,146 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:00:30,146 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:00:32,305 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:37,306 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:42,307 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:45,180 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:00:45,181 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:00:45,182 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:00:48,297 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:52,126 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 21:00:54,127 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:00:59,128 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:00,468 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:01:00,469 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:01:00,509 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:01:04,611 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:09,612 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:14,613 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:15,554 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:01:15,554 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:01:15,594 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:01:19,645 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:22,126 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 21:01:25,128 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:30,129 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:30,502 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:01:30,503 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:01:30,542 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:01:35,673 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:40,674 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:45,502 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:01:45,503 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:01:45,543 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:01:46,640 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:51,641 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:01:52,127 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 21:01:57,129 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:00,502 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:02:00,502 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:02:00,542 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:02:02,602 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:07,603 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:12,604 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:15,502 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:02:15,503 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:02:15,543 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:02:17,656 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:22,128 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 21:02:23,129 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:28,130 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:30,647 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:02:30,647 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:02:30,687 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:02:33,732 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:38,733 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:43,734 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:45,595 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:02:45,596 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:02:45,596 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:02:49,716 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:02:52,129 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 21:02:55,130 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:03:00,131 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:03:00,595 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:03:00,596 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:03:00,596 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:03:05,677 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:03:10,678 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:03:10,695 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: partial_history +2024-05-24 21:03:10,697 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 21:03:10,698 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 21:03:10,698 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 21:03:10,698 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 21:03:10,698 DEBUG SenderThread:1201 [sender.py:send():379] send: history +2024-05-24 21:03:10,698 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: summary_record +2024-05-24 21:03:10,698 INFO SenderThread:1201 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-24 21:03:11,254 INFO Thread-12 :1201 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/wandb-summary.json +2024-05-24 21:03:15,700 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:03:15,790 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:03:15,791 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:03:15,796 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:03:20,823 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: partial_history +2024-05-24 21:03:20,825 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 21:03:20,826 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 21:03:20,826 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 21:03:20,826 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 21:03:20,826 DEBUG SenderThread:1201 [sender.py:send():379] send: metric +2024-05-24 21:03:20,826 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:03:20,827 DEBUG SenderThread:1201 [sender.py:send():379] send: history +2024-05-24 21:03:20,827 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: summary_record +2024-05-24 21:03:20,828 INFO SenderThread:1201 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-24 21:03:21,258 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/wandb-summary.json +2024-05-24 21:03:22,130 DEBUG SenderThread:1201 [sender.py:send():379] send: stats +2024-05-24 21:03:25,259 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/output.log +2024-05-24 21:03:26,400 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:03:30,765 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: stop_status +2024-05-24 21:03:30,766 DEBUG SenderThread:1201 [sender.py:send_request():406] send_request: stop_status +2024-05-24 21:03:30,806 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-24 21:03:31,920 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report +2024-05-24 21:03:32,262 INFO Thread-12 :1201 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240524_202737-n1w0kmmv/files/config.yaml +2024-05-24 21:03:37,080 DEBUG HandlerThread:1201 [handler.py:handle_request():146] handle_request: status_report