diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" new file mode 100644--- /dev/null +++ "b/wandb/debug-internal.log" @@ -0,0 +1,2025 @@ +2024-02-03 23:15:35,314 INFO StreamThr :124 [internal.py:wandb_internal():86] W&B internal server running at pid: 124, started at: 2024-02-03 23:15:35.313611 +2024-02-03 23:15:35,315 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status +2024-02-03 23:15:35,627 INFO WriterThread:124 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/run-ot9nmopj.wandb +2024-02-03 23:15:35,628 DEBUG SenderThread:124 [sender.py:send():382] send: header +2024-02-03 23:15:35,631 DEBUG SenderThread:124 [sender.py:send():382] send: run +2024-02-03 23:15:35,787 INFO SenderThread:124 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files +2024-02-03 23:15:35,787 INFO SenderThread:124 [sender.py:_start_run_threads():1136] run started: ot9nmopj with start time 1707002135.31502 +2024-02-03 23:15:35,794 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: check_version +2024-02-03 23:15:35,795 DEBUG SenderThread:124 [sender.py:send_request():409] send_request: check_version +2024-02-03 23:15:35,882 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: run_start +2024-02-03 23:15:35,893 DEBUG HandlerThread:124 [system_info.py:__init__():27] System info init +2024-02-03 23:15:35,893 DEBUG HandlerThread:124 [system_info.py:__init__():42] System info init done +2024-02-03 23:15:35,893 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:15:35,894 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:15:35,894 INFO HandlerThread:124 [system_monitor.py:probe():214] Collecting system info +2024-02-03 23:15:35,895 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:15:35,895 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:15:35,896 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:15:35,897 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:15:35,897 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:15:35,908 DEBUG HandlerThread:124 [system_info.py:probe():151] Probing system +2024-02-03 23:15:35,910 DEBUG HandlerThread:124 [gitlib.py:_init_repo():56] git repository is invalid +2024-02-03 23:15:35,911 DEBUG HandlerThread:124 [system_info.py:probe():199] Probing system done +2024-02-03 23:15:35,911 DEBUG HandlerThread:124 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-02-03T23:15:35.908839', 'startedAt': '2024-02-03T23:15:35.307472', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '28d6b566919d', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.122, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.122, 'min': 0.0, 'max': 0.0}, {'current': 2000.122, 'min': 0.0, 'max': 0.0}, {'current': 2000.122, 'min': 0.0, 'max': 0.0}, {'current': 2000.122, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5432.328353881836}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-02-03 23:15:35,911 INFO HandlerThread:124 [system_monitor.py:probe():224] Finished collecting system info +2024-02-03 23:15:35,911 INFO HandlerThread:124 [system_monitor.py:probe():227] Publishing system info +2024-02-03 23:15:35,911 DEBUG HandlerThread:124 [system_info.py:_save_conda():208] Saving list of conda packages installed into the current environment +2024-02-03 23:15:36,789 INFO Thread-12 :124 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/conda-environment.yaml +2024-02-03 23:16:31,804 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/conda-environment.yaml +2024-02-03 23:16:32,028 DEBUG HandlerThread:124 [system_info.py:_save_conda():220] Saving conda packages done +2024-02-03 23:16:32,029 INFO HandlerThread:124 [system_monitor.py:probe():229] Finished publishing system info +2024-02-03 23:16:32,038 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,038 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,038 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,038 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,038 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,038 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,038 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: python_packages +2024-02-03 23:16:32,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: stop_status +2024-02-03 23:16:32,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: internal_messages +2024-02-03 23:16:32,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,040 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,040 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,040 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,040 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,041 DEBUG SenderThread:124 [sender.py:send_request():409] send_request: python_packages +2024-02-03 23:16:32,042 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,043 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,043 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:32,043 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:32,044 DEBUG SenderThread:124 [sender.py:send():382] send: telemetry +2024-02-03 23:16:32,044 DEBUG SenderThread:124 [sender.py:send():382] send: config +2024-02-03 23:16:32,045 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-03 23:16:32,046 DEBUG SenderThread:124 [sender.py:send():382] send: telemetry +2024-02-03 23:16:32,046 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-03 23:16:32,046 WARNING SenderThread:124 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-03 23:16:32,046 DEBUG SenderThread:124 [sender.py:send():382] send: telemetry +2024-02-03 23:16:32,046 DEBUG SenderThread:124 [sender.py:send_request():409] send_request: stop_status +2024-02-03 23:16:32,136 DEBUG SenderThread:124 [sender.py:send():382] send: files +2024-02-03 23:16:32,137 INFO SenderThread:124 [sender.py:_save_file():1403] saving file wandb-metadata.json with policy now +2024-02-03 23:16:32,389 INFO wandb-upload_0:124 [upload_job.py:push():131] Uploaded file /tmp/tmp02hps8r1wandb/7q6mpaux-wandb-metadata.json +2024-02-03 23:16:32,805 INFO Thread-12 :124 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/output.log +2024-02-03 23:16:32,805 INFO Thread-12 :124 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/requirements.txt +2024-02-03 23:16:32,805 INFO Thread-12 :124 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/wandb-metadata.json +2024-02-03 23:16:34,806 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/output.log +2024-02-03 23:16:35,898 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:16:35,899 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:16:35,906 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:35,937 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:36,807 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/config.yaml +2024-02-03 23:16:40,938 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:41,592 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:45,939 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:46,593 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:50,940 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:51,594 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:16:55,941 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:16:56,595 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:00,942 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:01,596 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:05,900 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:17:05,943 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:06,901 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:10,944 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:11,902 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:15,957 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:16,903 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:20,963 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:21,904 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:25,964 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:26,904 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:30,965 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:31,905 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:35,901 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:17:35,966 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:37,902 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:40,966 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:42,903 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:45,967 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:47,904 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:50,968 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:52,905 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:17:55,969 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:17:57,905 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:00,970 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:02,906 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:05,902 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:18:05,971 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:08,903 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:10,972 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:13,904 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:15,973 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:18,905 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:20,979 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:23,906 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:25,980 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:28,906 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:30,981 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:33,907 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:35,903 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:18:35,982 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:39,904 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:40,984 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:42,428 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:18:42,428 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:18:42,428 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:18:42,429 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:18:42,429 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:18:42,430 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:18:42,430 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:18:42,441 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:18:42,442 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:18:42,442 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:18:42,442 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:18:45,443 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:45,990 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:50,444 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:50,991 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:18:55,445 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:18:55,992 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:00,446 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:00,993 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:05,447 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:05,994 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:10,448 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:10,995 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:15,449 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:15,996 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:20,450 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:20,997 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:25,451 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:25,998 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:30,452 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:31,000 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:35,453 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:36,001 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:40,454 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:41,002 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:45,455 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:46,003 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:50,456 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:51,004 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:19:55,457 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:19:56,005 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:00,458 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:01,006 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:05,459 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:06,007 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:10,460 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:11,008 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:12,569 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:20:12,569 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:20:12,569 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:20:12,569 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:20:12,570 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:20:12,570 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:20:12,572 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:20:12,572 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:20:12,573 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:20:12,573 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:20:12,573 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:20:12,573 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:20:12,573 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:20:12,576 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:20:12,577 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:20:12,596 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:20:12,597 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:20:15,598 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:16,009 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:20,598 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:21,010 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:25,600 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:26,011 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:26,690 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:20:26,690 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:20:26,690 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:20:26,690 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:20:26,691 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:20:26,692 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:20:26,692 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:20:26,698 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:20:26,699 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:20:27,639 DEBUG SenderThread:124 [sender.py:send():382] send: config +2024-02-03 23:20:27,641 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-03 23:20:27,641 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-03 23:20:27,641 WARNING SenderThread:124 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-03 23:20:30,642 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:31,390 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:35,643 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:36,391 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:40,649 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:40,894 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/config.yaml +2024-02-03 23:20:41,400 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:45,752 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:46,401 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:50,753 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:51,408 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:20:55,754 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:20:56,409 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:00,755 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:01,410 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:05,755 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:06,411 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:10,756 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:11,412 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:15,757 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:16,416 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:20,758 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:21,417 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:25,759 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:26,418 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:26,699 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:21:26,700 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:21:31,419 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:31,702 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:36,420 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:36,702 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:41,421 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:41,703 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:46,428 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:46,704 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:51,429 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:51,705 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:21:56,430 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:21:56,701 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:21:57,702 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:01,431 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:02,703 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:06,440 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:07,704 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:11,452 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:12,705 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:16,453 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:17,705 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:21,454 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:22,706 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:26,455 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:26,702 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:22:28,703 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:31,457 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:33,704 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:36,458 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:38,705 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:41,459 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:43,705 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:46,460 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:48,706 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:51,461 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:53,707 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:22:56,464 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:22:56,703 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:22:59,704 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:01,465 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:04,236 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:23:04,236 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:23:04,236 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:23:04,237 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:23:04,237 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:23:04,238 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:23:04,239 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:23:04,250 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:23:04,251 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:23:04,251 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:23:04,251 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:23:05,252 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:06,466 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:10,253 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:11,467 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:15,254 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:16,467 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:20,254 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:21,468 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:25,255 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:26,469 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:30,256 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:31,470 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:35,257 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:36,472 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:40,258 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:41,473 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:45,259 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:46,474 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:50,260 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:51,475 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:23:55,261 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:23:56,476 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:00,261 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:01,477 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:05,262 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:06,478 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:10,263 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:11,479 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:15,264 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:16,480 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:20,265 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:21,481 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:25,266 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:26,482 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:30,267 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:31,483 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:35,267 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:36,484 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:40,268 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:41,485 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:45,269 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:46,486 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:50,270 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:51,487 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:24:55,271 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:24:56,488 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:00,271 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:01,489 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:05,272 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:06,490 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:10,273 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:11,491 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:15,274 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:16,492 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:20,275 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:21,493 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:25,276 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:26,494 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:30,277 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:31,495 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:35,278 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:36,496 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:40,279 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:41,497 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:45,279 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:46,498 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:50,280 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:51,499 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:25:55,281 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:25:56,500 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:00,282 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:01,501 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:05,283 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:06,502 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:10,284 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:11,503 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:15,285 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:16,504 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:20,285 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:21,505 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:25,286 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:26,506 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:30,287 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:31,507 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:35,288 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:36,508 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:40,288 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:41,510 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:45,289 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:46,510 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:50,290 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:51,512 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:26:55,291 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:26:56,513 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:00,291 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:01,514 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:05,292 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:06,515 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:10,293 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:11,516 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:15,294 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:16,517 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:20,295 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:21,518 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:25,296 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:26,519 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:30,296 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:31,520 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:35,297 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:36,521 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:40,298 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:41,522 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:45,299 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:46,523 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:50,300 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:51,524 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:55,301 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:27:56,525 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:27:59,262 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:27:59,262 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:27:59,262 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:27:59,262 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:27:59,263 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:27:59,263 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:27:59,264 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:27:59,265 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:27:59,265 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:27:59,265 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:27:59,265 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:27:59,265 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:27:59,266 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:27:59,267 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:27:59,269 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:27:59,289 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:27:59,290 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:28:01,291 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:01,526 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:02,866 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:28:02,866 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:28:02,866 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:28:02,867 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:28:02,867 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:28:02,868 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:28:02,869 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:28:02,869 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:28:02,870 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:28:04,130 DEBUG SenderThread:124 [sender.py:send():382] send: config +2024-02-03 23:28:04,131 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-03 23:28:04,131 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-03 23:28:04,131 WARNING SenderThread:124 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-03 23:28:06,865 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:07,132 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:11,866 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:12,132 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:16,867 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:17,139 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:18,056 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/config.yaml +2024-02-03 23:28:21,868 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:22,240 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:26,869 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:27,240 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:31,870 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:32,241 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:36,871 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:37,242 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:41,872 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:42,243 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:46,873 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:47,244 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:51,874 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:52,245 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:28:56,875 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:28:57,245 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:01,876 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:02,246 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:02,870 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:29:02,871 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:29:06,877 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:07,873 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:11,889 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:12,873 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:16,890 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:17,874 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:21,891 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:22,875 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:26,892 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:27,876 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:31,893 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:32,872 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:29:33,873 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:36,894 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:38,874 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:41,895 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:43,875 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:46,896 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:48,876 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:51,898 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:53,877 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:29:56,899 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:29:58,878 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:01,900 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:02,873 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:30:04,874 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:06,901 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:09,875 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:11,902 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:14,876 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:16,902 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:19,877 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:21,907 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:24,878 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:26,910 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:29,880 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:29,990 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:30:29,990 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:30:29,990 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:30:29,990 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:30:29,991 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:30:29,992 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:30:29,992 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:30:30,002 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:30:30,003 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:30:30,003 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:30:30,008 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:30:31,918 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:35,010 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:36,919 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:36,919 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:40,011 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:41,920 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:45,012 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:46,921 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:50,013 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:51,922 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:30:55,014 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:30:56,923 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:00,015 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:01,924 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:05,016 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:06,925 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:10,017 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:11,926 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:15,018 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:16,927 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:20,018 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:21,928 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:25,019 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:26,929 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:30,020 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:31,930 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:35,021 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:36,931 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:40,022 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:41,932 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:45,023 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:46,933 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:50,024 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:51,934 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:31:55,025 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:31:56,934 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:00,026 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:01,935 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:05,027 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:06,936 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:10,028 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:11,937 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:15,029 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:16,938 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:20,030 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:21,939 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:25,031 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:26,940 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:30,032 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:31,941 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:35,033 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:36,942 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:40,033 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:41,943 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:45,034 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:46,944 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:50,035 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:51,945 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:32:55,036 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:32:56,946 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:00,037 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:01,947 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:05,038 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:06,948 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:10,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:11,949 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:15,040 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:16,950 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:20,041 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:21,951 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:25,041 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:26,953 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:30,042 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:31,954 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:35,043 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:36,955 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:40,044 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:41,956 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:45,045 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:46,957 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:50,046 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:51,958 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:33:55,047 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:33:56,959 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:00,048 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:01,960 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:05,049 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:06,961 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:10,050 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:11,962 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:15,051 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:16,963 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:20,052 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:21,964 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:25,052 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:26,965 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:30,053 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:31,966 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:35,054 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:36,967 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:40,055 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:41,968 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:45,056 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:46,969 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:50,057 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:51,970 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:34:55,058 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:34:56,971 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:00,059 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:01,972 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:05,060 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:06,973 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:10,061 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:11,974 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:15,062 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:16,975 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:20,063 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:21,976 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:25,064 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:26,977 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:30,065 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:31,978 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:35,066 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:36,979 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:40,067 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:41,979 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:45,067 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:46,980 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:50,068 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:51,981 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:35:55,069 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:35:56,982 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:00,070 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:01,983 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:05,071 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:06,984 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:10,072 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:11,985 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:15,073 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:16,986 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:20,074 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:21,987 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:25,075 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:26,988 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:29,934 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:36:29,934 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:36:29,934 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:36:29,934 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:36:29,935 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:36:29,936 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:36:29,938 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:36:29,938 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:36:29,939 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:36:29,939 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:36:29,939 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:36:29,939 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:36:29,940 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:36:29,940 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:36:29,940 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:36:29,940 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:36:29,964 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:36:29,965 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:36:29,965 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:36:30,966 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:31,989 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:35,967 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:36,990 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:40,968 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:41,991 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:45,968 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:46,992 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:50,969 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:51,993 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:36:55,970 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:36:56,994 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:00,896 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:37:00,896 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:37:00,896 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:37:00,896 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:37:00,897 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:37:00,897 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:37:00,898 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:37:00,898 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:37:00,899 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:37:00,955 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:37:00,955 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:37:00,955 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:37:00,955 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:37:00,956 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:37:00,956 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:37:00,957 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:37:00,957 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:37:00,972 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:37:00,972 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:37:00,972 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:37:00,972 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:00,973 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:37:01,995 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:05,974 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:06,996 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:09,909 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:37:09,909 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:37:09,909 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:37:09,910 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:37:09,910 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:37:09,911 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:37:09,913 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:37:09,914 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:37:09,915 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:37:09,958 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:37:09,958 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:37:09,958 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:37:09,961 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:37:09,961 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:37:09,961 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:37:09,961 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:37:09,962 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:37:09,975 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:37:09,976 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:37:09,976 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:37:09,976 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:37:10,977 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:11,997 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:15,978 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:16,998 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:20,979 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:21,999 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:25,980 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:27,000 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:30,981 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:32,001 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:35,982 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:37,002 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:37,205 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:37:37,206 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:37:37,206 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:37:37,206 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:37:37,206 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:37:37,207 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:37:37,209 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:37:37,210 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:37:37,210 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:37:37,211 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:37:37,211 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:37:37,210 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:37:37,212 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:37:37,211 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:37:37,212 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:37:37,212 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:37:37,213 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:37:37,234 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:37:37,235 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:37:37,235 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:37:37,235 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:37:41,236 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:42,003 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:46,237 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:47,004 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:48,586 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:37:48,586 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:37:48,586 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:37:48,587 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:37:48,587 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:37:48,588 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:37:48,589 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:37:48,590 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:37:48,590 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:37:48,633 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:37:48,633 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:37:48,633 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:37:48,633 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:37:48,634 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:37:48,634 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:37:48,635 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:37:48,635 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:37:48,647 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:37:48,647 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:37:48,647 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:37:48,648 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:37:51,648 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:52,005 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:37:52,934 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:37:52,934 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:37:52,934 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:37:52,934 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:37:52,935 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:37:52,936 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:37:52,936 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:37:52,938 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:37:52,938 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:37:52,938 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:37:52,938 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:37:52,938 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:37:52,939 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:37:52,939 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:37:52,940 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:37:52,942 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:37:52,961 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:37:52,961 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:37:52,962 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:37:56,963 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:37:57,006 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:01,964 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:02,007 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:06,965 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:07,008 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:11,966 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:12,009 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:16,967 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:17,010 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:21,967 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:22,011 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:26,968 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:27,012 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:31,969 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:32,013 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:32,254 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:38:32,254 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:38:32,254 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:38:32,255 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:38:32,255 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:38:32,256 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:38:32,257 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:38:32,258 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:38:32,258 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:38:32,259 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:38:32,259 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:38:32,259 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:38:32,260 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:38:32,260 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:38:32,261 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:38:32,261 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:38:32,264 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:38:32,282 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:38:32,283 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:38:32,283 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:38:32,283 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:38:37,014 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:37,284 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:42,015 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:42,285 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:47,016 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:47,286 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:52,020 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:52,287 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:38:57,020 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:38:57,288 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:02,022 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:02,289 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:07,022 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:07,289 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:12,024 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:12,290 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:17,025 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:17,291 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:22,026 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:22,292 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:22,780 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:39:22,780 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:39:22,780 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:39:22,780 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:39:22,781 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:39:22,781 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:39:22,783 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:39:22,784 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:39:22,786 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:39:22,836 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:39:22,836 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:39:22,836 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:39:22,837 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:39:22,837 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:39:22,837 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:39:22,837 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:39:22,838 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:39:22,849 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:39:22,850 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:39:22,850 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:39:22,850 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:39:27,027 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:27,852 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:32,028 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:32,852 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:37,029 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:37,853 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:42,030 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:42,854 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:47,031 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:47,855 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:52,032 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:52,856 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:39:57,033 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:39:57,857 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:02,034 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:02,858 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:07,035 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:07,859 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:12,036 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:12,860 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:17,037 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:17,861 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:22,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:22,862 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:27,040 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:27,863 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:32,041 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:32,864 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:37,042 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:37,865 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:42,043 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:42,865 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:47,044 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:47,866 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:52,045 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:52,867 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:40:57,046 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:40:57,868 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:02,047 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:02,869 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:07,048 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:07,870 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:10,509 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:41:10,510 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:41:10,510 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:41:10,510 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:41:10,511 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:41:10,511 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:41:10,512 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:41:10,512 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:41:10,513 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:41:10,589 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:41:10,589 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:41:10,589 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:41:10,590 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:41:10,590 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:41:10,590 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:41:10,591 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:41:10,591 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:41:10,601 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:41:10,602 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:41:10,602 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:41:10,602 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:41:12,049 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:13,603 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:17,050 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:18,604 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:22,051 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:23,605 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:27,052 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:28,606 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:32,053 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:33,607 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:37,054 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:38,608 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:42,055 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:43,609 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:47,056 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:48,610 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:52,057 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:53,611 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:41:57,058 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:41:58,612 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:02,059 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:03,613 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:07,060 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:08,614 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:12,061 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:13,615 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:17,062 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:18,616 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:22,063 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:23,617 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:27,064 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:28,618 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:32,065 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:33,619 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:37,066 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:38,620 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:42,066 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:43,621 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:47,068 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:48,623 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:52,069 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:53,623 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:42:57,070 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:42:58,624 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:02,071 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:03,625 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:07,072 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:08,626 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:12,073 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:13,627 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:17,074 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:18,628 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:22,075 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:23,629 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:27,076 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:28,630 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:32,077 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:33,631 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:37,078 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:38,632 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:42,079 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:43,633 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:47,080 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:48,634 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:52,081 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:53,635 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:43:57,081 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:43:58,636 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:02,083 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:03,636 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:07,084 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:08,637 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:12,085 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:13,638 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:17,086 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:18,639 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:22,087 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:23,640 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:27,088 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:28,641 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:32,089 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:33,642 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:37,090 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:38,643 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:42,091 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:43,644 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:47,092 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:48,645 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:52,093 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:53,646 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:44:57,093 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:44:58,647 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:02,095 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:03,648 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:07,096 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:08,649 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:12,097 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:13,650 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:17,098 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:18,651 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:22,099 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:23,652 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:27,100 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:28,652 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:32,101 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:33,653 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:37,102 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:38,654 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:42,103 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:43,655 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:47,104 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:48,656 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:52,105 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:53,657 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:45:57,106 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:45:58,658 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:02,106 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:03,659 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:07,107 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:08,660 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:12,108 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:13,661 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:17,109 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:18,661 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:22,110 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:23,662 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:27,112 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:28,663 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:32,113 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:33,664 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:37,114 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:38,665 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:42,115 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:43,666 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:47,116 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:48,667 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:52,117 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:53,668 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:46:57,118 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:46:58,669 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:02,119 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:03,670 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:07,120 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:08,671 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:12,121 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:13,672 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:17,122 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:18,673 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:22,123 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:23,674 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:27,124 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:28,675 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:32,125 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:33,676 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:37,126 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:38,677 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:42,127 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:43,678 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:47,128 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:48,678 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:52,129 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:53,680 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:47:57,131 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:47:58,681 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:02,132 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:03,682 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:07,133 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:08,683 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:12,134 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:13,684 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:17,135 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:18,685 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:22,136 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:23,685 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:27,137 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:28,686 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:32,138 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:33,687 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:37,139 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:38,688 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:42,140 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:43,689 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:47,141 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:48,690 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:52,142 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:53,691 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:48:57,143 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:48:58,692 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:02,144 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:03,693 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:07,145 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:08,693 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:12,146 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:13,694 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:17,147 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:18,695 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:22,148 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:23,696 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:27,149 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:28,697 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:32,150 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:33,698 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:37,151 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:38,699 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:40,500 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:49:40,500 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:49:40,500 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:49:40,500 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:49:40,501 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:49:40,502 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:49:40,504 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:49:40,506 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:49:40,510 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:49:40,586 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:49:40,586 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:49:40,586 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:49:40,587 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:49:40,587 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:49:40,587 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:49:40,587 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:49:40,588 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:49:40,600 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:49:40,600 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:49:40,600 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:49:40,601 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:49:42,152 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:44,602 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:45,814 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:49:45,814 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:49:45,814 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:49:45,814 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:49:45,815 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:49:45,815 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:49:45,816 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:49:45,817 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:49:45,817 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:49:45,863 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:49:45,863 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:49:45,863 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:49:45,863 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:49:45,864 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:49:45,864 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:49:45,864 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:49:45,864 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:49:45,876 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:49:45,877 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:49:45,877 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:49:45,877 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:49:47,153 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:49,878 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:52,154 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:54,879 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:49:57,155 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:49:59,880 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:02,157 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:04,881 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:07,158 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:09,882 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:12,159 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:14,883 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:17,160 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:19,883 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:22,161 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:24,884 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:27,162 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:29,885 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:32,163 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:34,886 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:37,164 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:39,887 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:42,165 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:44,888 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:47,166 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:49,889 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:52,167 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:54,890 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:50:57,168 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:50:59,891 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:02,169 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:04,892 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:07,170 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:09,893 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:12,172 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:14,894 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:17,015 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:51:17,016 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:51:17,016 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:51:17,016 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:51:17,016 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:51:17,017 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:51:17,019 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:51:17,019 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:51:17,020 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:51:17,071 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:51:17,071 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:51:17,071 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:51:17,071 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:51:17,071 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:51:17,072 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:51:17,072 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:51:17,072 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:51:17,084 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:51:17,085 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:51:17,085 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:51:17,085 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:51:17,173 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:20,086 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:22,174 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:23,374 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:51:23,375 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:51:23,375 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:51:23,375 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:51:23,375 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:51:23,376 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:51:23,377 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:51:23,381 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:51:23,382 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:51:23,383 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:51:23,383 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:51:23,384 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:51:23,384 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:51:23,384 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:51:23,384 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:51:23,385 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:51:23,387 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:51:23,408 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:51:23,409 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:51:23,409 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:51:23,409 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:51:25,410 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:27,175 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:30,411 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:32,176 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:35,412 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:37,177 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:40,413 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:42,178 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:45,414 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:47,179 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:50,415 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:52,180 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:51:55,416 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:51:57,181 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:00,417 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:02,182 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:05,418 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:07,183 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:10,418 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:12,184 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:15,419 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:17,185 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:20,420 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:22,186 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:25,421 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:27,187 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:30,422 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:32,188 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:35,422 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:37,189 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:40,423 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:42,190 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:45,424 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:47,191 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:50,425 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:52,192 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:52:55,426 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:52:57,193 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:00,427 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:02,194 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:05,427 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:07,195 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:10,428 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:12,196 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:14,314 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:53:14,314 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:53:14,314 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:53:14,314 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:53:14,314 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:53:14,315 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:53:14,317 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:53:14,318 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:53:14,318 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:53:14,324 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:53:14,324 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:53:14,324 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:53:14,325 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:53:14,325 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:53:14,325 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:53:14,325 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:53:14,325 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:53:14,342 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:53:14,342 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:53:14,342 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:53:14,343 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:53:16,344 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:17,197 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:21,345 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:22,198 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:26,345 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:27,199 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:31,346 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:32,200 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:36,347 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:37,201 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:41,348 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:42,202 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:46,349 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:47,203 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:51,350 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:52,204 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:53:56,351 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:53:57,205 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:01,352 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:02,206 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:06,353 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:07,207 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:11,354 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:12,208 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:16,355 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:17,209 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:21,356 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:22,210 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:26,356 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:27,211 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:31,357 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:32,212 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:36,358 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:37,213 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:41,359 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:42,214 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:46,360 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:47,215 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:51,361 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:52,216 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:54:56,362 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:54:57,217 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:01,363 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:02,218 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:06,364 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:07,219 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:11,365 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:12,220 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:16,365 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:17,221 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:21,366 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:22,222 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:26,367 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:27,223 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:31,368 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:32,224 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:36,369 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:37,225 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:41,370 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:42,226 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:46,370 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:47,227 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:51,371 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:52,228 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:55:56,372 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:55:57,229 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:01,373 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:02,230 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:06,374 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:07,231 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:11,375 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:12,232 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:16,375 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:17,233 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:21,376 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:22,234 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:26,377 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:27,235 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:31,378 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:32,236 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:36,379 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:37,237 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:41,380 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:42,238 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:46,380 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:47,239 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:51,381 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:52,240 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:56:56,382 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:56:57,241 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:01,383 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:02,242 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:06,384 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:07,243 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:11,385 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:12,244 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:16,386 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:17,244 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:21,386 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:22,245 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:26,118 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:57:26,118 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:57:26,118 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:57:26,118 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:57:26,119 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:57:26,120 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:57:26,120 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:57:26,121 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:57:26,122 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:57:26,192 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:57:26,192 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:57:26,192 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:57:26,193 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:57:26,193 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:57:26,193 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:57:26,194 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:57:26,194 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:57:26,205 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:57:26,205 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:57:26,205 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:57:26,205 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:57:27,206 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:27,246 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:32,207 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:32,247 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:37,208 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:37,248 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:38,256 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:57:38,256 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:57:38,256 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:57:38,256 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:57:38,257 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:57:38,257 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:57:38,258 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:57:38,260 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:57:38,261 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:57:38,262 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:57:38,263 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:57:38,263 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:57:38,264 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:57:38,264 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:57:38,264 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:57:38,264 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:57:38,266 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:57:38,286 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:57:38,286 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:57:38,286 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:57:38,286 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:57:42,249 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:42,287 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:47,250 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:47,288 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:52,251 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:52,289 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:57:53,698 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-03 23:57:53,698 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-03 23:57:53,698 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-03 23:57:53,698 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-03 23:57:53,699 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-03 23:57:53,700 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-03 23:57:53,701 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-03 23:57:53,701 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-03 23:57:53,703 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-03 23:57:53,703 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-03 23:57:53,703 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-03 23:57:53,703 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-03 23:57:53,704 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-03 23:57:53,704 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-03 23:57:53,704 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-03 23:57:53,706 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-03 23:57:53,708 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-03 23:57:53,728 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-03 23:57:53,728 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-03 23:57:53,728 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-03 23:57:53,728 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-03 23:57:57,252 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:57:57,729 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:02,253 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:02,730 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:07,254 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:07,731 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:12,255 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:12,732 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:17,256 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:17,732 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:22,258 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:22,733 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:27,259 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:27,734 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:32,259 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:32,735 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:37,260 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:37,736 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:42,261 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:42,737 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:47,262 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:47,738 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:52,263 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:52,739 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:58:57,264 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:58:57,740 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:02,265 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:02,740 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:07,266 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:07,741 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:12,267 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:12,742 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:17,267 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:17,743 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:22,268 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:22,744 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:27,269 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:27,745 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:32,270 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:32,746 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:37,271 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:37,747 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:42,272 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:42,747 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:47,273 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:47,748 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:52,274 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:52,749 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-03 23:59:57,275 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-03 23:59:57,750 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:02,276 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:02,751 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:07,277 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:07,751 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:12,278 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:12,753 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:17,279 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:17,754 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:22,280 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:22,754 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:27,280 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:27,755 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:32,281 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:32,756 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:32,826 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:00:32,827 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:00:32,827 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:00:32,827 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:00:32,828 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:00:32,828 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:00:32,829 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:00:32,829 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:00:32,830 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-04 00:00:32,881 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:00:32,881 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:00:32,881 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:00:32,881 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:00:32,881 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:00:32,882 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:00:32,882 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:00:32,883 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:00:32,896 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:00:32,896 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-04 00:00:32,896 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-04 00:00:32,897 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:00:37,282 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:37,898 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:42,283 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:42,899 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:44,272 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:00:44,272 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:00:44,272 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:00:44,272 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:00:44,274 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:00:44,274 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:00:44,276 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:00:44,277 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:00:44,280 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-04 00:00:44,328 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:00:44,328 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:00:44,328 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:00:44,330 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:00:44,330 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:00:44,330 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:00:44,332 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:00:44,332 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:00:44,342 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:00:44,342 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-04 00:00:44,342 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-04 00:00:44,342 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:00:46,753 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/output.log +2024-02-04 00:00:47,284 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:48,344 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:52,285 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:52,870 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:00:52,870 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:00:52,871 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:00:52,871 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:00:52,872 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:00:52,873 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:00:52,877 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:00:52,877 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:00:52,878 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-04 00:00:52,923 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:00:52,923 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:00:52,923 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:00:52,924 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:00:52,924 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:00:52,924 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:00:52,925 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:00:52,926 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:00:52,939 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:00:52,939 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-04 00:00:52,939 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-04 00:00:52,940 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:00:53,940 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:00:54,755 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/output.log +2024-02-04 00:00:57,286 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:00:58,941 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:02,287 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:03,942 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:07,288 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:08,943 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:12,290 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:13,944 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:17,291 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:18,945 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:22,292 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:23,945 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:27,293 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:28,946 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:32,293 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:33,947 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:37,294 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:38,948 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:42,295 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:43,949 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:47,296 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:48,563 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:01:48,563 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:01:48,563 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:01:48,564 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:01:48,564 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:01:48,565 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:01:48,566 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:01:48,568 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:01:48,570 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-04 00:01:48,617 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:01:48,617 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:01:48,617 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:01:48,617 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:01:48,618 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:01:48,618 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:01:48,618 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:01:48,618 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:01:48,630 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:01:48,630 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-04 00:01:48,631 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-04 00:01:48,631 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:01:49,632 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:50,775 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/output.log +2024-02-04 00:01:52,297 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:54,633 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:01:57,298 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:01:59,634 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:02,299 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:04,635 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:07,300 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:09,635 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:12,301 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:14,636 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:17,302 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:19,579 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:02:19,580 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:02:19,580 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:02:19,580 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:02:19,580 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:02:19,581 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:02:19,582 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:02:19,582 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:02:19,583 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-04 00:02:19,628 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:02:19,628 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:02:19,628 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:02:19,628 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:02:19,629 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:02:19,629 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:02:19,629 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:02:19,633 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:02:19,642 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:02:19,642 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-04 00:02:19,642 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-04 00:02:19,642 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:19,643 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:02:22,303 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:23,516 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:02:23,516 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:02:23,516 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:02:23,516 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:02:23,517 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:02:23,518 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:02:23,520 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:02:23,520 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:02:23,520 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:02:23,520 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:02:23,520 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:02:23,521 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:02:23,522 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:02:23,522 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:02:23,524 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:02:23,544 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:02:23,544 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:02:25,545 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:27,305 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:30,546 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:32,305 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:35,547 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:37,307 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:40,548 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:42,308 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:45,549 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:47,309 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:50,550 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:52,310 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:02:55,551 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:02:57,311 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:00,552 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:02,312 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:05,552 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:06,244 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:03:06,244 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:03:06,244 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:03:06,244 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:03:06,246 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:03:06,246 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:03:06,248 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:03:06,254 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:03:06,254 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:03:06,254 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:03:06,254 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:03:06,256 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:03:06,256 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:03:06,256 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:03:06,258 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:03:06,259 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:03:06,276 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:03:06,276 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-04 00:03:06,277 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:03:07,313 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:08,801 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/output.log +2024-02-04 00:03:11,278 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:12,314 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:16,279 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:17,315 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:21,280 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:22,316 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:26,281 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:27,317 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:31,282 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:32,318 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:36,283 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:37,319 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:41,284 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:42,320 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:46,285 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:47,321 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:51,286 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:52,322 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:03:56,286 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:03:57,323 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:01,287 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:02,324 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:06,288 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:07,325 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:11,289 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:12,326 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:13,136 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:04:13,136 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:04:13,136 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:04:13,137 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:04:13,138 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:04:13,138 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:04:13,139 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:04:13,139 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:04:13,139 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:04:13,139 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:04:13,139 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:04:13,139 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:04:13,140 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:04:13,140 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:04:13,141 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:04:17,142 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:17,328 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:20,007 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:04:20,007 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:04:20,007 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:04:20,007 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:04:20,008 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:04:20,009 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:04:20,010 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:04:20,010 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:04:20,011 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-04 00:04:20,014 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:04:20,014 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:04:20,014 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:04:20,014 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:04:20,015 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:04:20,015 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:04:20,016 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:04:20,017 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:04:20,033 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:04:20,033 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-04 00:04:20,033 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-04 00:04:20,034 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:04:22,329 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:23,035 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:27,330 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:28,035 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:32,331 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:33,036 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:37,332 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:38,037 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:42,333 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:43,038 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:47,334 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:48,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:52,335 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:53,039 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:04:53,748 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:04:53,748 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:04:53,748 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:04:53,749 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:04:53,749 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:04:53,750 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:04:53,750 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:04:53,751 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:04:53,752 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-04 00:04:54,361 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:04:54,361 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:04:54,361 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:04:54,361 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:04:54,362 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:04:54,362 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:04:54,363 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:04:54,363 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:04:54,374 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:04:54,375 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-04 00:04:54,375 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-04 00:04:54,375 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:04:57,341 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:04:57,854 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:04:57,855 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:04:57,855 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:04:57,855 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:04:57,856 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:04:57,856 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:04:57,857 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:04:57,859 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:04:57,860 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-04 00:04:57,860 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: pause +2024-02-04 00:04:57,860 INFO HandlerThread:124 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-02-04 00:04:57,860 INFO HandlerThread:124 [system_monitor.py:finish():203] Stopping system monitor +2024-02-04 00:04:57,862 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:04:57,863 DEBUG SystemMonitor:124 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-02-04 00:04:57,863 DEBUG SystemMonitor:124 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-02-04 00:04:57,864 INFO HandlerThread:124 [interfaces.py:finish():202] Joined cpu monitor +2024-02-04 00:04:57,866 INFO HandlerThread:124 [interfaces.py:finish():202] Joined disk monitor +2024-02-04 00:04:57,884 INFO HandlerThread:124 [interfaces.py:finish():202] Joined gpu monitor +2024-02-04 00:04:57,884 INFO HandlerThread:124 [interfaces.py:finish():202] Joined memory monitor +2024-02-04 00:04:57,884 INFO HandlerThread:124 [interfaces.py:finish():202] Joined network monitor +2024-02-04 00:04:57,885 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:04:58,886 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:02,037 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: resume +2024-02-04 00:05:02,037 INFO HandlerThread:124 [handler.py:handle_request_resume():699] starting system metrics thread +2024-02-04 00:05:02,037 INFO HandlerThread:124 [system_monitor.py:start():194] Starting system monitor +2024-02-04 00:05:02,038 INFO SystemMonitor:124 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-02-04 00:05:02,038 INFO SystemMonitor:124 [interfaces.py:start():190] Started cpu monitoring +2024-02-04 00:05:02,039 INFO SystemMonitor:124 [interfaces.py:start():190] Started disk monitoring +2024-02-04 00:05:02,040 INFO SystemMonitor:124 [interfaces.py:start():190] Started gpu monitoring +2024-02-04 00:05:02,042 INFO SystemMonitor:124 [interfaces.py:start():190] Started memory monitoring +2024-02-04 00:05:02,046 INFO SystemMonitor:124 [interfaces.py:start():190] Started network monitoring +2024-02-04 00:05:02,826 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:03,132 DEBUG SenderThread:124 [sender.py:send():382] send: config +2024-02-04 00:05:03,134 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:05:03,134 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:05:03,134 WARNING SenderThread:124 [sender.py:send_metric():1354] Seen metric with glob (shouldn't happen) +2024-02-04 00:05:04,288 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:04,841 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/config.yaml +2024-02-04 00:05:05,841 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/output.log +2024-02-04 00:05:07,827 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:09,432 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:12,828 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:14,433 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:17,829 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:19,434 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:22,843 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:24,435 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:27,844 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:29,436 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:32,845 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:34,437 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:37,846 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:39,438 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:42,850 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:44,439 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:47,851 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:49,439 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:52,852 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:54,440 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:05:57,853 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:05:59,441 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:02,046 DEBUG SystemMonitor:124 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-02-04 00:06:02,048 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:06:02,854 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:05,049 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:07,855 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:10,050 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:12,856 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:15,051 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:17,857 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:20,052 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:22,858 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:25,052 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:27,859 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:30,053 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:32,050 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:06:32,861 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:36,052 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:37,862 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:41,053 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:42,863 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:46,054 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:47,864 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:51,055 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:52,865 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:06:56,055 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:06:57,865 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:01,056 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:02,050 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:07:02,867 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:07,051 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:07,868 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:12,052 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:12,869 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:17,053 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:17,869 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:22,054 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:22,876 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:27,055 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:27,879 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:32,051 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:07:32,887 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:33,051 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:37,903 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:38,052 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:43,053 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:43,224 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:47,897 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/output.log +2024-02-04 00:07:48,258 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:48,880 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:53,288 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:53,881 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:07:54,848 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: partial_history +2024-02-04 00:07:54,852 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:07:54,852 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:07:54,852 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:07:54,853 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:07:54,853 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:07:54,853 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:07:54,853 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:07:54,853 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:07:54,853 DEBUG SenderThread:124 [sender.py:send():382] send: metric +2024-02-04 00:07:54,853 DEBUG SenderThread:124 [sender.py:send():382] send: history +2024-02-04 00:07:54,853 DEBUG SenderThread:124 [sender.py:send_request():409] send_request: summary_record +2024-02-04 00:07:54,854 INFO SenderThread:124 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end +2024-02-04 00:07:54,899 INFO Thread-12 :124 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/wandb-summary.json +2024-02-04 00:07:57,901 INFO Thread-12 :124 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240203_231535-ot9nmopj/files/output.log +2024-02-04 00:07:58,380 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive +2024-02-04 00:07:59,519 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: status_report +2024-02-04 00:08:02,052 DEBUG SenderThread:124 [sender.py:send():382] send: stats +2024-02-04 00:08:03,381 DEBUG HandlerThread:124 [handler.py:handle_request():146] handle_request: keepalive