diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,697 +1,248 @@ -2023-12-21 18:33:10,472 INFO StreamThr :505 [internal.py:wandb_internal():86] W&B internal server running at pid: 505, started at: 2023-12-21 18:33:10.471481 -2023-12-21 18:33:10,473 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status -2023-12-21 18:33:10,966 INFO WriterThread:505 [datastore.py:open_for_write():85] open: /kaggle/working/wandb/run-20231221_183310-x7g99l13/run-x7g99l13.wandb -2023-12-21 18:33:10,966 DEBUG SenderThread:505 [sender.py:send():382] send: header -2023-12-21 18:33:10,969 DEBUG SenderThread:505 [sender.py:send():382] send: run -2023-12-21 18:33:11,164 INFO SenderThread:505 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files -2023-12-21 18:33:11,165 INFO SenderThread:505 [sender.py:_start_run_threads():1136] run started: x7g99l13 with start time 1703183590.471415 -2023-12-21 18:33:11,172 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: check_version -2023-12-21 18:33:11,173 DEBUG SenderThread:505 [sender.py:send_request():409] send_request: check_version -2023-12-21 18:33:11,270 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: run_start -2023-12-21 18:33:11,274 DEBUG HandlerThread:505 [system_info.py:__init__():32] System info init -2023-12-21 18:33:11,274 DEBUG HandlerThread:505 [system_info.py:__init__():47] System info init done -2023-12-21 18:33:11,275 INFO HandlerThread:505 [system_monitor.py:start():194] Starting system monitor -2023-12-21 18:33:11,275 INFO SystemMonitor:505 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-12-21 18:33:11,275 INFO HandlerThread:505 [system_monitor.py:probe():214] Collecting system info -2023-12-21 18:33:11,276 INFO SystemMonitor:505 [interfaces.py:start():190] Started cpu monitoring -2023-12-21 18:33:11,276 INFO SystemMonitor:505 [interfaces.py:start():190] Started disk monitoring -2023-12-21 18:33:11,277 INFO SystemMonitor:505 [interfaces.py:start():190] Started gpu monitoring -2023-12-21 18:33:11,278 INFO SystemMonitor:505 [interfaces.py:start():190] Started memory monitoring -2023-12-21 18:33:11,279 INFO SystemMonitor:505 [interfaces.py:start():190] Started network monitoring -2023-12-21 18:33:11,290 DEBUG HandlerThread:505 [system_info.py:probe():196] Probing system -2023-12-21 18:33:11,293 DEBUG HandlerThread:505 [gitlib.py:_init_repo():56] git repository is invalid -2023-12-21 18:33:11,293 DEBUG HandlerThread:505 [system_info.py:probe():244] Probing system done -2023-12-21 18:33:11,293 DEBUG HandlerThread:505 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2023-12-21T18:33:11.290750', 'startedAt': '2023-12-21T18:33:10.465127', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '66c15c1844b5', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.182, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.182, 'min': 0.0, 'max': 0.0}, {'current': 2000.182, 'min': 0.0, 'max': 0.0}, {'current': 2000.182, 'min': 0.0, 'max': 0.0}, {'current': 2000.182, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5310.905750274658}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 15843721216}, {'name': 'Tesla T4', 'memory_total': 15843721216}], 'memory': {'total': 31.357559204101562}} -2023-12-21 18:33:11,293 INFO HandlerThread:505 [system_monitor.py:probe():224] Finished collecting system info -2023-12-21 18:33:11,293 INFO HandlerThread:505 [system_monitor.py:probe():227] Publishing system info -2023-12-21 18:33:11,293 DEBUG HandlerThread:505 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment -2023-12-21 18:33:11,295 DEBUG HandlerThread:505 [system_info.py:_save_pip():68] Saving pip packages done -2023-12-21 18:33:11,295 DEBUG HandlerThread:505 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment -2023-12-21 18:33:12,167 INFO Thread-12 :505 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/requirements.txt -2023-12-21 18:33:12,167 INFO Thread-12 :505 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/conda-environment.yaml -2023-12-21 18:34:00,182 INFO Thread-12 :505 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/conda-environment.yaml -2023-12-21 18:34:00,688 DEBUG HandlerThread:505 [system_info.py:_save_conda():87] Saving conda packages done -2023-12-21 18:34:00,688 INFO HandlerThread:505 [system_monitor.py:probe():229] Finished publishing system info -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:00,694 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:00,695 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: stop_status -2023-12-21 18:34:00,695 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: internal_messages -2023-12-21 18:34:00,695 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:00,695 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:00,695 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: partial_history -2023-12-21 18:34:00,697 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: pause -2023-12-21 18:34:00,697 INFO HandlerThread:505 [handler.py:handle_request_pause():705] stopping system metrics thread -2023-12-21 18:34:00,697 INFO HandlerThread:505 [system_monitor.py:finish():203] Stopping system monitor -2023-12-21 18:34:00,698 DEBUG SystemMonitor:505 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2023-12-21 18:34:00,698 DEBUG SystemMonitor:505 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2023-12-21 18:34:00,698 DEBUG SystemMonitor:505 [system_monitor.py:_start():183] Publishing last batch of metrics -2023-12-21 18:34:00,699 DEBUG SenderThread:505 [sender.py:send_request():409] send_request: stop_status -2023-12-21 18:34:00,702 INFO HandlerThread:505 [interfaces.py:finish():202] Joined cpu monitor -2023-12-21 18:34:00,703 INFO HandlerThread:505 [interfaces.py:finish():202] Joined disk monitor -2023-12-21 18:34:00,717 INFO HandlerThread:505 [interfaces.py:finish():202] Joined gpu monitor -2023-12-21 18:34:00,717 INFO HandlerThread:505 [interfaces.py:finish():202] Joined memory monitor -2023-12-21 18:34:00,717 INFO HandlerThread:505 [interfaces.py:finish():202] Joined network monitor -2023-12-21 18:34:00,717 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: resume -2023-12-21 18:34:00,717 INFO HandlerThread:505 [handler.py:handle_request_resume():696] starting system metrics thread -2023-12-21 18:34:00,717 INFO HandlerThread:505 [system_monitor.py:start():194] Starting system monitor -2023-12-21 18:34:00,717 INFO SystemMonitor:505 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-12-21 18:34:00,718 INFO SystemMonitor:505 [interfaces.py:start():190] Started cpu monitoring -2023-12-21 18:34:00,718 INFO SystemMonitor:505 [interfaces.py:start():190] Started disk monitoring -2023-12-21 18:34:00,719 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:00,719 INFO SystemMonitor:505 [interfaces.py:start():190] Started gpu monitoring -2023-12-21 18:34:00,719 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:00,720 INFO SystemMonitor:505 [interfaces.py:start():190] Started memory monitoring -2023-12-21 18:34:00,720 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: pause -2023-12-21 18:34:00,721 INFO SystemMonitor:505 [interfaces.py:start():190] Started network monitoring -2023-12-21 18:34:00,721 INFO HandlerThread:505 [handler.py:handle_request_pause():705] stopping system metrics thread -2023-12-21 18:34:00,722 INFO HandlerThread:505 [system_monitor.py:finish():203] Stopping system monitor -2023-12-21 18:34:00,722 INFO HandlerThread:505 [interfaces.py:finish():202] Joined cpu monitor -2023-12-21 18:34:00,722 DEBUG SystemMonitor:505 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2023-12-21 18:34:00,722 DEBUG SystemMonitor:505 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2023-12-21 18:34:00,722 DEBUG SystemMonitor:505 [system_monitor.py:_start():183] Publishing last batch of metrics -2023-12-21 18:34:00,725 INFO HandlerThread:505 [interfaces.py:finish():202] Joined disk monitor -2023-12-21 18:34:00,749 INFO HandlerThread:505 [interfaces.py:finish():202] Joined gpu monitor -2023-12-21 18:34:00,749 INFO HandlerThread:505 [interfaces.py:finish():202] Joined memory monitor -2023-12-21 18:34:00,749 INFO HandlerThread:505 [interfaces.py:finish():202] Joined network monitor -2023-12-21 18:34:00,749 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: resume -2023-12-21 18:34:00,750 INFO HandlerThread:505 [handler.py:handle_request_resume():696] starting system metrics thread -2023-12-21 18:34:00,750 INFO HandlerThread:505 [system_monitor.py:start():194] Starting system monitor -2023-12-21 18:34:00,750 INFO SystemMonitor:505 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-12-21 18:34:00,750 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:00,750 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:00,750 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: pause -2023-12-21 18:34:00,751 INFO HandlerThread:505 [handler.py:handle_request_pause():705] stopping system metrics thread -2023-12-21 18:34:00,751 INFO HandlerThread:505 [system_monitor.py:finish():203] Stopping system monitor -2023-12-21 18:34:00,751 INFO SystemMonitor:505 [interfaces.py:start():190] Started cpu monitoring -2023-12-21 18:34:00,751 DEBUG SystemMonitor:505 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2023-12-21 18:34:00,751 DEBUG SystemMonitor:505 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2023-12-21 18:34:00,751 DEBUG SystemMonitor:505 [system_monitor.py:_start():183] Publishing last batch of metrics -2023-12-21 18:34:00,752 INFO HandlerThread:505 [interfaces.py:finish():202] Joined cpu monitor -2023-12-21 18:34:00,788 DEBUG SenderThread:505 [sender.py:send():382] send: telemetry -2023-12-21 18:34:00,788 DEBUG SenderThread:505 [sender.py:send():382] send: config -2023-12-21 18:34:00,789 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:34:00,790 DEBUG SenderThread:505 [sender.py:send():382] send: telemetry -2023-12-21 18:34:00,790 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:34:00,790 WARNING SenderThread:505 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) -2023-12-21 18:34:00,790 DEBUG SenderThread:505 [sender.py:send():382] send: telemetry -2023-12-21 18:34:00,791 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:34:00,791 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:34:00,791 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:34:00,791 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:34:00,791 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:34:00,791 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:34:00,791 DEBUG SenderThread:505 [sender.py:send():382] send: history -2023-12-21 18:34:00,791 DEBUG SenderThread:505 [sender.py:send_request():409] send_request: summary_record -2023-12-21 18:34:00,793 INFO SenderThread:505 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2023-12-21 18:34:00,793 DEBUG SenderThread:505 [sender.py:send():382] send: files -2023-12-21 18:34:00,793 INFO SenderThread:505 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now -2023-12-21 18:34:00,794 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:34:00,794 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:34:00,794 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:34:01,007 INFO wandb-upload_0:505 [upload_job.py:push():131] Uploaded file /tmp/tmpp2ayfjkjwandb/gfdnvxn3-wandb-metadata.json -2023-12-21 18:34:01,183 INFO Thread-12 :505 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/output.log -2023-12-21 18:34:01,183 INFO Thread-12 :505 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/wandb-summary.json -2023-12-21 18:34:01,183 INFO Thread-12 :505 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/wandb-metadata.json -2023-12-21 18:34:01,795 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:02,582 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:03,184 INFO Thread-12 :505 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/output.log -2023-12-21 18:34:06,797 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:07,583 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:11,803 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:12,188 INFO Thread-12 :505 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/config.yaml -2023-12-21 18:34:12,584 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:16,938 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:17,585 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:21,939 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:22,586 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:26,940 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:27,587 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:31,941 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:32,589 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:36,942 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:37,590 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:41,943 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:42,591 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:46,944 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:47,592 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:51,945 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:52,593 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:34:56,947 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:34:57,594 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:01,948 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:02,595 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:06,949 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:07,597 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:11,950 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:12,598 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:16,951 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:17,599 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:21,952 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:22,600 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:26,953 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:27,602 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:31,954 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:32,603 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:36,956 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:37,604 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:41,957 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:42,606 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:46,958 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:47,607 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:51,959 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:52,608 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:35:56,960 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:35:57,609 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:01,961 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:02,610 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:06,962 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:07,611 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:11,963 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:12,613 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:16,964 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:17,614 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:21,965 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:22,616 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:26,966 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:27,617 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:31,968 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:32,618 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:36,968 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:37,619 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:41,970 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:42,621 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:46,971 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:47,622 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:51,972 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:52,623 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:36:56,974 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:36:57,625 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:01,975 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:02,626 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:06,976 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:07,627 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:11,977 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:12,628 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:16,978 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:17,629 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:21,979 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:22,631 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:26,980 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:27,632 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:31,982 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:32,633 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:36,983 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:37,635 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:41,984 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:42,636 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:46,985 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:47,637 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:51,986 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:52,639 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:37:56,987 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:37:57,640 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:01,988 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:02,641 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:06,989 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:07,642 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:11,990 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:12,644 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:16,991 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:17,645 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:21,992 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:22,646 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:26,993 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:27,648 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:31,994 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:32,649 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:36,995 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:37,650 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:41,997 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:42,651 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:46,998 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:47,653 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:51,999 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:52,654 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:38:57,000 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:38:57,655 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:02,001 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:02,656 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:07,002 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:07,658 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:12,004 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:12,659 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:17,005 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:17,660 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:22,006 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:22,661 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:27,007 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:27,663 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:32,008 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:32,664 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:37,009 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:37,666 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:42,010 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:42,667 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:47,011 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:47,668 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:52,012 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:52,670 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:39:57,013 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:39:57,671 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:02,014 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:02,672 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:07,015 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:07,673 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:12,016 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:12,675 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:17,018 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:17,676 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:22,019 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:22,677 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:27,020 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:27,678 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:32,021 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:32,680 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:37,022 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:37,681 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:42,023 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:42,682 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:47,024 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:47,683 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:52,025 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:52,685 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:40:57,026 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:40:57,686 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:02,027 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:02,687 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:07,028 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:07,688 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:12,029 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:12,689 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:17,030 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:17,691 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:19,086 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: resume -2023-12-21 18:41:19,086 INFO HandlerThread:505 [handler.py:handle_request_resume():696] starting system metrics thread -2023-12-21 18:41:19,086 INFO HandlerThread:505 [system_monitor.py:start():194] Starting system monitor -2023-12-21 18:41:19,086 INFO SystemMonitor:505 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-12-21 18:41:19,087 INFO SystemMonitor:505 [interfaces.py:start():190] Started cpu monitoring -2023-12-21 18:41:19,087 INFO SystemMonitor:505 [interfaces.py:start():190] Started disk monitoring -2023-12-21 18:41:19,088 INFO SystemMonitor:505 [interfaces.py:start():190] Started gpu monitoring -2023-12-21 18:41:19,089 INFO SystemMonitor:505 [interfaces.py:start():190] Started memory monitoring -2023-12-21 18:41:19,091 INFO SystemMonitor:505 [interfaces.py:start():190] Started network monitoring -2023-12-21 18:41:22,031 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:23,044 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:27,032 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:27,192 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: pause -2023-12-21 18:41:27,192 INFO HandlerThread:505 [handler.py:handle_request_pause():705] stopping system metrics thread -2023-12-21 18:41:27,192 INFO HandlerThread:505 [system_monitor.py:finish():203] Stopping system monitor -2023-12-21 18:41:27,193 INFO HandlerThread:505 [interfaces.py:finish():202] Joined cpu monitor -2023-12-21 18:41:27,193 INFO HandlerThread:505 [interfaces.py:finish():202] Joined disk monitor -2023-12-21 18:41:27,193 DEBUG SystemMonitor:505 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2023-12-21 18:41:27,193 DEBUG SystemMonitor:505 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2023-12-21 18:41:27,193 DEBUG SystemMonitor:505 [system_monitor.py:_start():183] Publishing last batch of metrics -2023-12-21 18:41:27,209 INFO HandlerThread:505 [interfaces.py:finish():202] Joined gpu monitor -2023-12-21 18:41:27,210 INFO HandlerThread:505 [interfaces.py:finish():202] Joined memory monitor -2023-12-21 18:41:27,210 INFO HandlerThread:505 [interfaces.py:finish():202] Joined network monitor -2023-12-21 18:41:27,210 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:41:28,046 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:32,212 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:33,047 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:36,524 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: resume -2023-12-21 18:41:36,524 INFO HandlerThread:505 [handler.py:handle_request_resume():696] starting system metrics thread -2023-12-21 18:41:36,524 INFO HandlerThread:505 [system_monitor.py:start():194] Starting system monitor -2023-12-21 18:41:36,524 INFO SystemMonitor:505 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-12-21 18:41:36,525 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: pause -2023-12-21 18:41:36,525 INFO HandlerThread:505 [handler.py:handle_request_pause():705] stopping system metrics thread -2023-12-21 18:41:36,525 INFO HandlerThread:505 [system_monitor.py:finish():203] Stopping system monitor -2023-12-21 18:41:36,525 WARNING HandlerThread:505 [interfaces.py:finish():207] Failed to finish cpu monitoring: cannot join thread before it is started -2023-12-21 18:41:36,525 INFO SystemMonitor:505 [interfaces.py:start():190] Started cpu monitoring -2023-12-21 18:41:36,525 DEBUG SystemMonitor:505 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2023-12-21 18:41:36,526 DEBUG SystemMonitor:505 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2023-12-21 18:41:36,526 DEBUG SystemMonitor:505 [system_monitor.py:_start():183] Publishing last batch of metrics -2023-12-21 18:41:36,527 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:41:37,527 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:38,048 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:38,217 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: resume -2023-12-21 18:41:38,217 INFO HandlerThread:505 [handler.py:handle_request_resume():696] starting system metrics thread -2023-12-21 18:41:38,217 INFO HandlerThread:505 [system_monitor.py:start():194] Starting system monitor -2023-12-21 18:41:38,217 INFO SystemMonitor:505 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-12-21 18:41:38,218 INFO SystemMonitor:505 [interfaces.py:start():190] Started cpu monitoring -2023-12-21 18:41:38,218 INFO SystemMonitor:505 [interfaces.py:start():190] Started disk monitoring -2023-12-21 18:41:38,219 INFO SystemMonitor:505 [interfaces.py:start():190] Started gpu monitoring -2023-12-21 18:41:38,222 INFO SystemMonitor:505 [interfaces.py:start():190] Started memory monitoring -2023-12-21 18:41:38,224 INFO SystemMonitor:505 [interfaces.py:start():190] Started network monitoring -2023-12-21 18:41:38,876 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: pause -2023-12-21 18:41:38,876 INFO HandlerThread:505 [handler.py:handle_request_pause():705] stopping system metrics thread -2023-12-21 18:41:38,876 INFO HandlerThread:505 [system_monitor.py:finish():203] Stopping system monitor -2023-12-21 18:41:38,876 INFO HandlerThread:505 [interfaces.py:finish():202] Joined cpu monitor -2023-12-21 18:41:38,877 DEBUG SystemMonitor:505 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2023-12-21 18:41:38,877 DEBUG SystemMonitor:505 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2023-12-21 18:41:38,877 INFO HandlerThread:505 [interfaces.py:finish():202] Joined disk monitor -2023-12-21 18:41:38,877 DEBUG SystemMonitor:505 [system_monitor.py:_start():183] Publishing last batch of metrics -2023-12-21 18:41:38,893 INFO HandlerThread:505 [interfaces.py:finish():202] Joined gpu monitor -2023-12-21 18:41:38,894 INFO HandlerThread:505 [interfaces.py:finish():202] Joined memory monitor -2023-12-21 18:41:38,894 INFO HandlerThread:505 [interfaces.py:finish():202] Joined network monitor -2023-12-21 18:41:38,894 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:41:42,895 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:43,049 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:47,896 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:48,050 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:52,897 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:53,051 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:41:57,898 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:41:58,053 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:02,899 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:03,054 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:07,900 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:08,055 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:12,901 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:13,056 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:17,901 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:18,058 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:21,610 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: resume -2023-12-21 18:42:21,610 INFO HandlerThread:505 [handler.py:handle_request_resume():696] starting system metrics thread -2023-12-21 18:42:21,610 INFO HandlerThread:505 [system_monitor.py:start():194] Starting system monitor -2023-12-21 18:42:21,611 INFO SystemMonitor:505 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-12-21 18:42:21,611 INFO SystemMonitor:505 [interfaces.py:start():190] Started cpu monitoring -2023-12-21 18:42:21,612 INFO SystemMonitor:505 [interfaces.py:start():190] Started disk monitoring -2023-12-21 18:42:21,613 INFO SystemMonitor:505 [interfaces.py:start():190] Started gpu monitoring -2023-12-21 18:42:21,615 INFO SystemMonitor:505 [interfaces.py:start():190] Started memory monitoring -2023-12-21 18:42:21,615 INFO SystemMonitor:505 [interfaces.py:start():190] Started network monitoring -2023-12-21 18:42:22,186 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: pause -2023-12-21 18:42:22,187 INFO HandlerThread:505 [handler.py:handle_request_pause():705] stopping system metrics thread -2023-12-21 18:42:22,187 INFO HandlerThread:505 [system_monitor.py:finish():203] Stopping system monitor -2023-12-21 18:42:22,187 DEBUG SystemMonitor:505 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2023-12-21 18:42:22,187 DEBUG SystemMonitor:505 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2023-12-21 18:42:22,187 DEBUG SystemMonitor:505 [system_monitor.py:_start():183] Publishing last batch of metrics -2023-12-21 18:42:22,189 INFO HandlerThread:505 [interfaces.py:finish():202] Joined cpu monitor -2023-12-21 18:42:22,189 INFO HandlerThread:505 [interfaces.py:finish():202] Joined disk monitor -2023-12-21 18:42:22,204 INFO HandlerThread:505 [interfaces.py:finish():202] Joined gpu monitor -2023-12-21 18:42:22,205 INFO HandlerThread:505 [interfaces.py:finish():202] Joined memory monitor -2023-12-21 18:42:22,205 INFO HandlerThread:505 [interfaces.py:finish():202] Joined network monitor -2023-12-21 18:42:22,205 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:42:23,059 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:23,206 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:28,060 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:28,207 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:33,061 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:33,208 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:38,063 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:38,210 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:43,064 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:43,211 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:48,065 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:48,212 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:53,067 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:53,213 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:42:58,068 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:42:58,214 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:03,069 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:03,216 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:08,070 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:08,217 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:13,071 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:13,218 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:18,072 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:18,219 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:23,073 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:23,220 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:28,075 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:28,221 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:33,076 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:33,223 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:38,078 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:38,224 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:43,079 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:43,225 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:48,081 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:48,226 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:53,082 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:53,227 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:43:58,083 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:43:58,229 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:03,085 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:03,230 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:08,086 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:08,231 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:13,087 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:13,232 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:18,088 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:18,233 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:23,090 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:23,235 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:28,091 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:28,236 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:33,092 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:33,237 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:38,094 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:38,238 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:43,095 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:43,239 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:48,096 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:48,240 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:53,098 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:53,241 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:44:58,099 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:44:58,242 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:03,100 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:03,243 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:08,101 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:08,244 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:13,102 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:13,245 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:18,104 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:18,247 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:23,105 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:23,248 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:28,106 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:28,249 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:33,108 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:33,250 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:38,109 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:38,251 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:43,110 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:43,252 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:48,112 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:48,253 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:53,113 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:53,255 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:45:58,114 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:45:58,256 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:03,115 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:03,257 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:08,117 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:08,258 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:13,118 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:13,259 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:18,119 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:18,261 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:23,120 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:23,262 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:28,122 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:28,263 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:33,123 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:33,265 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:38,125 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:38,266 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:43,126 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:43,267 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:48,127 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:48,268 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:53,128 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:53,269 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:46:58,129 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:46:58,270 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:03,130 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:03,271 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:08,131 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:08,272 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:13,133 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:13,273 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:18,135 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:18,275 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:23,136 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:23,276 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:28,137 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:28,277 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:33,138 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:33,278 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:38,140 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:38,280 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:43,141 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:43,281 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:48,142 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:48,282 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:53,144 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:53,284 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:47:58,145 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:47:58,285 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:03,146 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:03,286 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:08,147 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:08,287 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:13,148 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:13,289 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:18,149 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:18,290 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:23,151 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:23,292 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:28,152 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:28,293 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:33,153 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:33,294 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:38,154 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:38,295 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:43,155 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:43,296 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:48,157 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:48,298 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:53,158 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:53,299 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:48:58,159 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:48:58,300 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:03,160 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:03,301 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:08,161 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:08,302 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:13,163 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:13,303 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:18,164 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:18,305 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:23,165 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:23,306 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:28,167 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:28,307 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:33,168 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:33,308 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:38,168 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:38,309 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:43,170 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:43,311 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:48,171 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:48,312 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:53,172 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:53,313 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:49:58,173 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:49:58,314 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:03,174 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:03,315 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:05,513 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: resume -2023-12-21 18:50:05,513 INFO HandlerThread:505 [handler.py:handle_request_resume():696] starting system metrics thread -2023-12-21 18:50:05,513 INFO HandlerThread:505 [system_monitor.py:start():194] Starting system monitor -2023-12-21 18:50:05,514 INFO SystemMonitor:505 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-12-21 18:50:05,514 INFO SystemMonitor:505 [interfaces.py:start():190] Started cpu monitoring -2023-12-21 18:50:05,515 INFO SystemMonitor:505 [interfaces.py:start():190] Started disk monitoring -2023-12-21 18:50:05,516 INFO SystemMonitor:505 [interfaces.py:start():190] Started gpu monitoring -2023-12-21 18:50:05,517 INFO SystemMonitor:505 [interfaces.py:start():190] Started memory monitoring -2023-12-21 18:50:05,518 INFO SystemMonitor:505 [interfaces.py:start():190] Started network monitoring -2023-12-21 18:50:06,070 DEBUG SenderThread:505 [sender.py:send():382] send: config -2023-12-21 18:50:06,071 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:50:06,072 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:50:06,072 WARNING SenderThread:505 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) -2023-12-21 18:50:08,175 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:08,586 INFO Thread-12 :505 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/output.log -2023-12-21 18:50:09,152 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:13,176 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:14,158 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:14,589 INFO Thread-12 :505 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/config.yaml -2023-12-21 18:50:18,177 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:19,351 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:23,178 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:24,352 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:28,179 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:29,353 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:33,181 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:34,354 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:38,183 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:39,355 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:43,184 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:44,356 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:48,185 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:49,357 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:53,186 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:54,358 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:50:58,187 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:50:59,359 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:03,188 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:04,360 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:05,518 DEBUG SystemMonitor:505 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2023-12-21 18:51:05,519 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:51:08,189 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:09,521 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:13,195 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:14,522 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:18,196 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:19,522 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:23,197 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:24,523 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:28,198 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:29,524 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:33,199 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:34,525 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:35,520 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:51:38,200 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:40,521 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:43,204 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:45,522 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:48,210 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:50,523 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:53,213 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:51:55,524 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:51:58,216 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:52:00,525 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:52:03,217 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:52:05,521 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:52:06,522 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:52:08,219 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:52:11,523 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:52:13,220 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:52:16,524 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:52:18,221 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:52:18,322 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: partial_history -2023-12-21 18:52:18,323 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:52:18,324 DEBUG SenderThread:505 [sender.py:send():382] send: metric -2023-12-21 18:52:18,324 DEBUG SenderThread:505 [sender.py:send():382] send: history -2023-12-21 18:52:18,324 DEBUG SenderThread:505 [sender.py:send_request():409] send_request: summary_record -2023-12-21 18:52:18,325 INFO SenderThread:505 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2023-12-21 18:52:18,635 INFO Thread-12 :505 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/wandb-summary.json -2023-12-21 18:52:20,237 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: partial_history -2023-12-21 18:52:20,239 DEBUG SenderThread:505 [sender.py:send():382] send: history -2023-12-21 18:52:20,239 DEBUG SenderThread:505 [sender.py:send_request():409] send_request: summary_record -2023-12-21 18:52:20,239 INFO SenderThread:505 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end -2023-12-21 18:52:20,636 INFO Thread-12 :505 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_183310-x7g99l13/files/wandb-summary.json -2023-12-21 18:52:22,240 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:52:23,271 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:52:27,242 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:52:28,272 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive -2023-12-21 18:52:32,243 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: status_report -2023-12-21 18:52:32,523 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: pause -2023-12-21 18:52:32,523 INFO HandlerThread:505 [handler.py:handle_request_pause():705] stopping system metrics thread -2023-12-21 18:52:32,523 INFO HandlerThread:505 [system_monitor.py:finish():203] Stopping system monitor -2023-12-21 18:52:32,523 DEBUG SystemMonitor:505 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2023-12-21 18:52:32,524 DEBUG SystemMonitor:505 [system_monitor.py:_start():183] Publishing last batch of metrics -2023-12-21 18:52:32,524 INFO HandlerThread:505 [interfaces.py:finish():202] Joined cpu monitor -2023-12-21 18:52:32,525 INFO HandlerThread:505 [interfaces.py:finish():202] Joined disk monitor -2023-12-21 18:52:32,541 INFO HandlerThread:505 [interfaces.py:finish():202] Joined gpu monitor -2023-12-21 18:52:32,542 INFO HandlerThread:505 [interfaces.py:finish():202] Joined memory monitor -2023-12-21 18:52:32,542 INFO HandlerThread:505 [interfaces.py:finish():202] Joined network monitor -2023-12-21 18:52:32,542 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: resume -2023-12-21 18:52:32,542 INFO HandlerThread:505 [handler.py:handle_request_resume():696] starting system metrics thread -2023-12-21 18:52:32,543 INFO HandlerThread:505 [system_monitor.py:start():194] Starting system monitor -2023-12-21 18:52:32,543 INFO SystemMonitor:505 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-12-21 18:52:32,543 INFO SystemMonitor:505 [interfaces.py:start():190] Started cpu monitoring -2023-12-21 18:52:32,544 DEBUG SenderThread:505 [sender.py:send():382] send: stats -2023-12-21 18:52:32,545 INFO SystemMonitor:505 [interfaces.py:start():190] Started disk monitoring -2023-12-21 18:52:32,546 INFO SystemMonitor:505 [interfaces.py:start():190] Started gpu monitoring -2023-12-21 18:52:32,546 INFO SystemMonitor:505 [interfaces.py:start():190] Started memory monitoring -2023-12-21 18:52:32,547 INFO SystemMonitor:505 [interfaces.py:start():190] Started network monitoring -2023-12-21 18:52:33,366 DEBUG HandlerThread:505 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:42:23,301 INFO StreamThr :204 [internal.py:wandb_internal():86] W&B internal server running at pid: 204, started at: 2023-12-21 19:42:23.300478 +2023-12-21 19:42:23,302 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status +2023-12-21 19:42:23,829 INFO WriterThread:204 [datastore.py:open_for_write():85] open: /kaggle/working/wandb/run-20231221_194223-lb3081d4/run-lb3081d4.wandb +2023-12-21 19:42:23,830 DEBUG SenderThread:204 [sender.py:send():382] send: header +2023-12-21 19:42:23,833 DEBUG SenderThread:204 [sender.py:send():382] send: run +2023-12-21 19:42:24,025 INFO SenderThread:204 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files +2023-12-21 19:42:24,025 INFO SenderThread:204 [sender.py:_start_run_threads():1136] run started: lb3081d4 with start time 1703187743.30193 +2023-12-21 19:42:24,032 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: check_version +2023-12-21 19:42:24,033 DEBUG SenderThread:204 [sender.py:send_request():409] send_request: check_version +2023-12-21 19:42:24,131 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: run_start +2023-12-21 19:42:24,135 DEBUG HandlerThread:204 [system_info.py:__init__():32] System info init +2023-12-21 19:42:24,136 DEBUG HandlerThread:204 [system_info.py:__init__():47] System info init done +2023-12-21 19:42:24,136 INFO HandlerThread:204 [system_monitor.py:start():194] Starting system monitor +2023-12-21 19:42:24,136 INFO SystemMonitor:204 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-12-21 19:42:24,136 INFO HandlerThread:204 [system_monitor.py:probe():214] Collecting system info +2023-12-21 19:42:24,136 INFO SystemMonitor:204 [interfaces.py:start():190] Started cpu monitoring +2023-12-21 19:42:24,137 INFO SystemMonitor:204 [interfaces.py:start():190] Started disk monitoring +2023-12-21 19:42:24,139 INFO SystemMonitor:204 [interfaces.py:start():190] Started gpu monitoring +2023-12-21 19:42:24,139 INFO SystemMonitor:204 [interfaces.py:start():190] Started memory monitoring +2023-12-21 19:42:24,140 INFO SystemMonitor:204 [interfaces.py:start():190] Started network monitoring +2023-12-21 19:42:24,156 DEBUG HandlerThread:204 [system_info.py:probe():196] Probing system +2023-12-21 19:42:24,158 DEBUG HandlerThread:204 [gitlib.py:_init_repo():56] git repository is invalid +2023-12-21 19:42:24,158 DEBUG HandlerThread:204 [system_info.py:probe():244] Probing system done +2023-12-21 19:42:24,158 DEBUG HandlerThread:204 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.35', 'python': '3.10.12', 'heartbeatAt': '2023-12-21T19:42:24.156703', 'startedAt': '2023-12-21T19:42:23.294100', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'fa65ad7b7892', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.136, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.136, 'min': 0.0, 'max': 0.0}, {'current': 2000.136, 'min': 0.0, 'max': 0.0}, {'current': 2000.136, 'min': 0.0, 'max': 0.0}, {'current': 2000.136, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5310.897075653076}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 15843721216}, {'name': 'Tesla T4', 'memory_total': 15843721216}], 'memory': {'total': 31.357559204101562}} +2023-12-21 19:42:24,159 INFO HandlerThread:204 [system_monitor.py:probe():224] Finished collecting system info +2023-12-21 19:42:24,159 INFO HandlerThread:204 [system_monitor.py:probe():227] Publishing system info +2023-12-21 19:42:24,159 DEBUG HandlerThread:204 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment +2023-12-21 19:42:24,160 DEBUG HandlerThread:204 [system_info.py:_save_pip():68] Saving pip packages done +2023-12-21 19:42:24,160 DEBUG HandlerThread:204 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment +2023-12-21 19:42:25,027 INFO Thread-12 :204 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/conda-environment.yaml +2023-12-21 19:42:25,027 INFO Thread-12 :204 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/requirements.txt +2023-12-21 19:43:15,898 DEBUG HandlerThread:204 [system_info.py:_save_conda():87] Saving conda packages done +2023-12-21 19:43:15,899 INFO HandlerThread:204 [system_monitor.py:probe():229] Finished publishing system info +2023-12-21 19:43:15,907 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,907 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:15,907 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,907 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:15,907 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,908 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:15,908 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,908 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:15,908 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,908 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:15,908 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,908 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: internal_messages +2023-12-21 19:43:15,908 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: stop_status +2023-12-21 19:43:15,909 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,909 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:15,909 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,910 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:15,910 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: partial_history +2023-12-21 19:43:15,912 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: pause +2023-12-21 19:43:15,912 INFO HandlerThread:204 [handler.py:handle_request_pause():705] stopping system metrics thread +2023-12-21 19:43:15,912 INFO HandlerThread:204 [system_monitor.py:finish():203] Stopping system monitor +2023-12-21 19:43:15,912 DEBUG SenderThread:204 [sender.py:send():382] send: telemetry +2023-12-21 19:43:15,912 DEBUG SenderThread:204 [sender.py:send_request():409] send_request: stop_status +2023-12-21 19:43:15,914 INFO HandlerThread:204 [interfaces.py:finish():202] Joined cpu monitor +2023-12-21 19:43:15,915 INFO HandlerThread:204 [interfaces.py:finish():202] Joined disk monitor +2023-12-21 19:43:15,917 DEBUG SystemMonitor:204 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-12-21 19:43:15,917 DEBUG SystemMonitor:204 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-12-21 19:43:15,917 DEBUG SystemMonitor:204 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-12-21 19:43:15,930 INFO HandlerThread:204 [interfaces.py:finish():202] Joined gpu monitor +2023-12-21 19:43:15,931 INFO HandlerThread:204 [interfaces.py:finish():202] Joined memory monitor +2023-12-21 19:43:15,931 INFO HandlerThread:204 [interfaces.py:finish():202] Joined network monitor +2023-12-21 19:43:15,931 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: resume +2023-12-21 19:43:15,931 INFO HandlerThread:204 [handler.py:handle_request_resume():696] starting system metrics thread +2023-12-21 19:43:15,931 INFO HandlerThread:204 [system_monitor.py:start():194] Starting system monitor +2023-12-21 19:43:15,931 INFO SystemMonitor:204 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-12-21 19:43:15,932 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: pause +2023-12-21 19:43:15,932 INFO HandlerThread:204 [handler.py:handle_request_pause():705] stopping system metrics thread +2023-12-21 19:43:15,932 INFO HandlerThread:204 [system_monitor.py:finish():203] Stopping system monitor +2023-12-21 19:43:15,932 INFO SystemMonitor:204 [interfaces.py:start():190] Started cpu monitoring +2023-12-21 19:43:15,932 DEBUG SystemMonitor:204 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-12-21 19:43:15,932 DEBUG SystemMonitor:204 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-12-21 19:43:15,932 DEBUG SystemMonitor:204 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-12-21 19:43:15,933 INFO HandlerThread:204 [interfaces.py:finish():202] Joined cpu monitor +2023-12-21 19:43:15,933 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,933 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:15,933 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:15,933 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:15,943 DEBUG SenderThread:204 [sender.py:send():382] send: config +2023-12-21 19:43:15,944 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:43:15,944 DEBUG SenderThread:204 [sender.py:send():382] send: telemetry +2023-12-21 19:43:15,945 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:43:15,945 WARNING SenderThread:204 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) +2023-12-21 19:43:15,945 DEBUG SenderThread:204 [sender.py:send():382] send: telemetry +2023-12-21 19:43:15,945 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:43:15,945 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:43:15,945 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:43:15,946 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:43:15,946 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:43:15,946 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:43:15,946 DEBUG SenderThread:204 [sender.py:send():382] send: history +2023-12-21 19:43:15,946 DEBUG SenderThread:204 [sender.py:send_request():409] send_request: summary_record +2023-12-21 19:43:15,948 INFO SenderThread:204 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2023-12-21 19:43:15,948 DEBUG SenderThread:204 [sender.py:send():382] send: files +2023-12-21 19:43:15,948 INFO SenderThread:204 [sender.py:_save_file():1392] saving file wandb-metadata.json with policy now +2023-12-21 19:43:15,949 DEBUG SenderThread:204 [sender.py:send():382] send: stats +2023-12-21 19:43:15,949 DEBUG SenderThread:204 [sender.py:send():382] send: stats +2023-12-21 19:43:16,042 INFO Thread-12 :204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/conda-environment.yaml +2023-12-21 19:43:16,042 INFO Thread-12 :204 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/wandb-summary.json +2023-12-21 19:43:16,042 INFO Thread-12 :204 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/output.log +2023-12-21 19:43:16,042 INFO Thread-12 :204 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/wandb-metadata.json +2023-12-21 19:43:16,139 INFO wandb-upload_0:204 [upload_job.py:push():131] Uploaded file /tmp/tmppl00ajsswandb/qt1h04ki-wandb-metadata.json +2023-12-21 19:43:18,042 INFO Thread-12 :204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/output.log +2023-12-21 19:43:19,170 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:19,944 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: resume +2023-12-21 19:43:19,945 INFO HandlerThread:204 [handler.py:handle_request_resume():696] starting system metrics thread +2023-12-21 19:43:19,945 INFO HandlerThread:204 [system_monitor.py:start():194] Starting system monitor +2023-12-21 19:43:19,945 INFO SystemMonitor:204 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-12-21 19:43:19,946 INFO SystemMonitor:204 [interfaces.py:start():190] Started cpu monitoring +2023-12-21 19:43:19,946 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: pause +2023-12-21 19:43:19,946 INFO HandlerThread:204 [handler.py:handle_request_pause():705] stopping system metrics thread +2023-12-21 19:43:19,946 INFO HandlerThread:204 [system_monitor.py:finish():203] Stopping system monitor +2023-12-21 19:43:19,947 INFO HandlerThread:204 [interfaces.py:finish():202] Joined cpu monitor +2023-12-21 19:43:19,947 INFO HandlerThread:204 [interfaces.py:finish():202] Joined disk monitor +2023-12-21 19:43:19,948 INFO SystemMonitor:204 [interfaces.py:start():190] Started disk monitoring +2023-12-21 19:43:19,948 DEBUG SystemMonitor:204 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-12-21 19:43:19,948 DEBUG SystemMonitor:204 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-12-21 19:43:19,948 DEBUG SystemMonitor:204 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-12-21 19:43:19,949 DEBUG SenderThread:204 [sender.py:send():382] send: stats +2023-12-21 19:43:19,949 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:20,644 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: resume +2023-12-21 19:43:20,644 INFO HandlerThread:204 [handler.py:handle_request_resume():696] starting system metrics thread +2023-12-21 19:43:20,644 INFO HandlerThread:204 [system_monitor.py:start():194] Starting system monitor +2023-12-21 19:43:20,644 INFO SystemMonitor:204 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-12-21 19:43:20,645 INFO SystemMonitor:204 [interfaces.py:start():190] Started cpu monitoring +2023-12-21 19:43:20,645 INFO SystemMonitor:204 [interfaces.py:start():190] Started disk monitoring +2023-12-21 19:43:20,646 INFO SystemMonitor:204 [interfaces.py:start():190] Started gpu monitoring +2023-12-21 19:43:20,647 INFO SystemMonitor:204 [interfaces.py:start():190] Started memory monitoring +2023-12-21 19:43:20,648 INFO SystemMonitor:204 [interfaces.py:start():190] Started network monitoring +2023-12-21 19:43:23,290 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: pause +2023-12-21 19:43:23,290 INFO HandlerThread:204 [handler.py:handle_request_pause():705] stopping system metrics thread +2023-12-21 19:43:23,290 INFO HandlerThread:204 [system_monitor.py:finish():203] Stopping system monitor +2023-12-21 19:43:23,290 DEBUG SystemMonitor:204 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-12-21 19:43:23,291 DEBUG SystemMonitor:204 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2023-12-21 19:43:23,291 DEBUG SystemMonitor:204 [system_monitor.py:_start():183] Publishing last batch of metrics +2023-12-21 19:43:23,291 INFO HandlerThread:204 [interfaces.py:finish():202] Joined cpu monitor +2023-12-21 19:43:23,292 INFO HandlerThread:204 [interfaces.py:finish():202] Joined disk monitor +2023-12-21 19:43:23,305 INFO HandlerThread:204 [interfaces.py:finish():202] Joined gpu monitor +2023-12-21 19:43:23,305 INFO HandlerThread:204 [interfaces.py:finish():202] Joined memory monitor +2023-12-21 19:43:23,305 INFO HandlerThread:204 [interfaces.py:finish():202] Joined network monitor +2023-12-21 19:43:23,305 DEBUG SenderThread:204 [sender.py:send():382] send: stats +2023-12-21 19:43:24,175 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:25,311 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:26,045 INFO Thread-12 :204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/config.yaml +2023-12-21 19:43:29,176 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:30,414 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:34,177 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:35,415 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:39,178 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:40,416 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:44,179 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:45,417 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:49,180 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:50,418 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:54,181 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:43:55,418 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:43:59,182 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:00,419 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:04,183 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:05,420 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:09,184 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:10,421 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:12,839 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: resume +2023-12-21 19:44:12,839 INFO HandlerThread:204 [handler.py:handle_request_resume():696] starting system metrics thread +2023-12-21 19:44:12,839 INFO HandlerThread:204 [system_monitor.py:start():194] Starting system monitor +2023-12-21 19:44:12,839 INFO SystemMonitor:204 [system_monitor.py:_start():158] Starting system asset monitoring threads +2023-12-21 19:44:12,840 INFO SystemMonitor:204 [interfaces.py:start():190] Started cpu monitoring +2023-12-21 19:44:12,841 INFO SystemMonitor:204 [interfaces.py:start():190] Started disk monitoring +2023-12-21 19:44:12,842 INFO SystemMonitor:204 [interfaces.py:start():190] Started gpu monitoring +2023-12-21 19:44:12,845 INFO SystemMonitor:204 [interfaces.py:start():190] Started memory monitoring +2023-12-21 19:44:12,845 INFO SystemMonitor:204 [interfaces.py:start():190] Started network monitoring +2023-12-21 19:44:13,359 DEBUG SenderThread:204 [sender.py:send():382] send: config +2023-12-21 19:44:13,360 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:44:13,360 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:44:13,361 WARNING SenderThread:204 [sender.py:send_metric():1343] Seen metric with glob (shouldn't happen) +2023-12-21 19:44:14,062 INFO Thread-12 :204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/output.log +2023-12-21 19:44:14,250 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:15,423 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:19,250 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:20,423 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:24,256 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:25,429 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:26,066 INFO Thread-12 :204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/config.yaml +2023-12-21 19:44:29,257 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:30,571 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:34,258 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:35,571 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:39,259 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:40,572 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:44,260 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:45,573 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:49,261 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:50,574 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:54,262 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:44:55,575 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:44:59,263 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:00,576 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:04,266 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:05,576 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:09,267 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:10,577 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:12,846 DEBUG SystemMonitor:204 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2023-12-21 19:45:12,847 DEBUG SenderThread:204 [sender.py:send():382] send: stats +2023-12-21 19:45:14,275 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:15,848 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:19,278 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:20,849 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:24,279 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:25,850 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:29,280 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:30,851 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:34,281 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:35,852 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:39,282 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:40,853 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:42,848 DEBUG SenderThread:204 [sender.py:send():382] send: stats +2023-12-21 19:45:44,289 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:46,849 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:49,290 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:51,854 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:54,291 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:45:56,855 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:45:59,291 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:46:01,856 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:46:04,292 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:46:06,857 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:46:09,293 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:46:11,858 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:46:12,849 DEBUG SenderThread:204 [sender.py:send():382] send: stats +2023-12-21 19:46:14,294 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:46:17,850 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:46:18,057 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: partial_history +2023-12-21 19:46:18,058 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:46:18,059 DEBUG SenderThread:204 [sender.py:send():382] send: metric +2023-12-21 19:46:18,059 DEBUG SenderThread:204 [sender.py:send():382] send: history +2023-12-21 19:46:18,059 DEBUG SenderThread:204 [sender.py:send_request():409] send_request: summary_record +2023-12-21 19:46:18,059 INFO SenderThread:204 [sender.py:_save_file():1392] saving file wandb-summary.json with policy end +2023-12-21 19:46:18,109 INFO Thread-12 :204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/wandb-summary.json +2023-12-21 19:46:19,336 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:46:23,111 INFO Thread-12 :204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/output.log +2023-12-21 19:46:23,151 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:46:24,502 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:46:28,158 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report +2023-12-21 19:46:29,114 INFO Thread-12 :204 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20231221_194223-lb3081d4/files/config.yaml +2023-12-21 19:46:29,503 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: keepalive +2023-12-21 19:46:33,292 DEBUG HandlerThread:204 [handler.py:handle_request():146] handle_request: status_report