diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,32 +1,32 @@ -2024-04-30 13:43:30,429 INFO StreamThr :161 [internal.py:wandb_internal():86] W&B internal server running at pid: 161, started at: 2024-04-30 13:43:30.429000 -2024-04-30 13:43:30,431 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status -2024-04-30 13:43:30,931 INFO WriterThread:161 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/run-p3yujlfg.wandb -2024-04-30 13:43:30,932 DEBUG SenderThread:161 [sender.py:send():379] send: header -2024-04-30 13:43:30,935 DEBUG SenderThread:161 [sender.py:send():379] send: run -2024-04-30 13:43:31,062 INFO SenderThread:161 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files -2024-04-30 13:43:31,062 INFO SenderThread:161 [sender.py:_start_run_threads():1124] run started: p3yujlfg with start time 1714484610.430593 -2024-04-30 13:43:31,070 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: check_version -2024-04-30 13:43:31,070 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: check_version -2024-04-30 13:43:31,164 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: run_start -2024-04-30 13:43:31,176 DEBUG HandlerThread:161 [system_info.py:__init__():26] System info init -2024-04-30 13:43:31,176 DEBUG HandlerThread:161 [system_info.py:__init__():41] System info init done -2024-04-30 13:43:31,176 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor -2024-04-30 13:43:31,176 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-30 13:43:31,176 INFO HandlerThread:161 [system_monitor.py:probe():214] Collecting system info -2024-04-30 13:43:31,177 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring -2024-04-30 13:43:31,178 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring -2024-04-30 13:43:31,178 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring -2024-04-30 13:43:31,180 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring -2024-04-30 13:43:31,180 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring -2024-04-30 13:43:31,191 DEBUG HandlerThread:161 [system_info.py:probe():150] Probing system -2024-04-30 13:43:31,193 DEBUG HandlerThread:161 [gitlib.py:_init_repo():56] git repository is invalid -2024-04-30 13:43:31,194 DEBUG HandlerThread:161 [system_info.py:probe():198] Probing system done -2024-04-30 13:43:31,194 DEBUG HandlerThread:161 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-04-30T13:43:31.191665', 'startedAt': '2024-04-30T13:43:30.422591', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'fde755c1ca53', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.144, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.144, 'min': 0.0, 'max': 0.0}, {'current': 2000.144, 'min': 0.0, 'max': 0.0}, {'current': 2000.144, 'min': 0.0, 'max': 0.0}, {'current': 2000.144, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5605.322723388672}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} -2024-04-30 13:43:31,194 INFO HandlerThread:161 [system_monitor.py:probe():224] Finished collecting system info -2024-04-30 13:43:31,194 INFO HandlerThread:161 [system_monitor.py:probe():227] Publishing system info -2024-04-30 13:43:31,194 DEBUG HandlerThread:161 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment -2024-04-30 13:43:32,064 INFO Thread-12 :161 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/conda-environment.yaml -2024-04-30 13:43:46,209 ERROR HandlerThread:161 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-05-01 08:36:06,671 INFO StreamThr :138 [internal.py:wandb_internal():86] W&B internal server running at pid: 138, started at: 2024-05-01 08:36:06.670530 +2024-05-01 08:36:06,673 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status +2024-05-01 08:36:06,939 INFO WriterThread:138 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240501_083606-td31zvzm/run-td31zvzm.wandb +2024-05-01 08:36:06,939 DEBUG SenderThread:138 [sender.py:send():379] send: header +2024-05-01 08:36:06,943 DEBUG SenderThread:138 [sender.py:send():379] send: run +2024-05-01 08:36:07,197 INFO SenderThread:138 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files +2024-05-01 08:36:07,197 INFO SenderThread:138 [sender.py:_start_run_threads():1124] run started: td31zvzm with start time 1714552566.671469 +2024-05-01 08:36:07,206 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: check_version +2024-05-01 08:36:07,207 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: check_version +2024-05-01 08:36:07,309 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: run_start +2024-05-01 08:36:07,321 DEBUG HandlerThread:138 [system_info.py:__init__():26] System info init +2024-05-01 08:36:07,321 DEBUG HandlerThread:138 [system_info.py:__init__():41] System info init done +2024-05-01 08:36:07,321 INFO HandlerThread:138 [system_monitor.py:start():194] Starting system monitor +2024-05-01 08:36:07,322 INFO SystemMonitor:138 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-01 08:36:07,322 INFO HandlerThread:138 [system_monitor.py:probe():214] Collecting system info +2024-05-01 08:36:07,322 INFO SystemMonitor:138 [interfaces.py:start():190] Started cpu monitoring +2024-05-01 08:36:07,323 INFO SystemMonitor:138 [interfaces.py:start():190] Started disk monitoring +2024-05-01 08:36:07,324 INFO SystemMonitor:138 [interfaces.py:start():190] Started gpu monitoring +2024-05-01 08:36:07,325 INFO SystemMonitor:138 [interfaces.py:start():190] Started memory monitoring +2024-05-01 08:36:07,326 INFO SystemMonitor:138 [interfaces.py:start():190] Started network monitoring +2024-05-01 08:36:07,340 DEBUG HandlerThread:138 [system_info.py:probe():150] Probing system +2024-05-01 08:36:07,343 DEBUG HandlerThread:138 [gitlib.py:_init_repo():56] git repository is invalid +2024-05-01 08:36:07,343 DEBUG HandlerThread:138 [system_info.py:probe():198] Probing system done +2024-05-01 08:36:07,343 DEBUG HandlerThread:138 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-05-01T08:36:07.340972', 'startedAt': '2024-05-01T08:36:06.663946', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '09efed9bf36a', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.128, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.128, 'min': 0.0, 'max': 0.0}, {'current': 2000.128, 'min': 0.0, 'max': 0.0}, {'current': 2000.128, 'min': 0.0, 'max': 0.0}, {'current': 2000.128, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5605.304306030273}}, 'gpu': 'Tesla T4', 'gpu_count': 2, 'gpu_devices': [{'name': 'Tesla T4', 'memory_total': 16106127360}, {'name': 'Tesla T4', 'memory_total': 16106127360}], 'memory': {'total': 31.357559204101562}} +2024-05-01 08:36:07,343 INFO HandlerThread:138 [system_monitor.py:probe():224] Finished collecting system info +2024-05-01 08:36:07,343 INFO HandlerThread:138 [system_monitor.py:probe():227] Publishing system info +2024-05-01 08:36:07,343 DEBUG HandlerThread:138 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-01 08:36:08,199 INFO Thread-12 :138 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/conda-environment.yaml +2024-05-01 08:36:22,357 ERROR HandlerThread:138 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda subprocess.call( @@ -37,841 +37,1257 @@ Traceback (most recent call last): File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait raise TimeoutExpired(self.args, timeout) subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds -2024-04-30 13:43:46,214 DEBUG HandlerThread:161 [system_info.py:_save_conda():222] Saving conda packages done -2024-04-30 13:43:46,215 INFO HandlerThread:161 [system_monitor.py:probe():229] Finished publishing system info -2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: keepalive -2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: keepalive -2024-04-30 13:43:46,224 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:43:46,225 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: keepalive -2024-04-30 13:43:46,225 DEBUG SenderThread:161 [sender.py:send():379] send: files -2024-04-30 13:43:46,225 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now -2024-04-30 13:43:46,421 INFO wandb-upload_0:161 [upload_job.py:push():131] Uploaded file /tmp/tmp1w88py37wandb/09spuzjy-wandb-metadata.json -2024-04-30 13:43:47,068 INFO Thread-12 :161 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-metadata.json -2024-04-30 13:43:47,242 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: python_packages -2024-04-30 13:43:47,243 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: python_packages -2024-04-30 13:43:47,246 DEBUG SenderThread:161 [sender.py:send():379] send: telemetry -2024-04-30 13:43:47,258 DEBUG SenderThread:161 [sender.py:send():379] send: config -2024-04-30 13:43:47,260 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:43:47,260 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:43:47,261 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:43:47,261 DEBUG SenderThread:161 [sender.py:send():379] send: telemetry -2024-04-30 13:43:47,261 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:43:47,262 WARNING SenderThread:161 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) -2024-04-30 13:43:47,262 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:43:47,382 DEBUG SenderThread:161 [sender.py:send():379] send: telemetry -2024-04-30 13:43:48,068 INFO Thread-12 :161 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/requirements.txt -2024-04-30 13:43:51,383 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:43:56,384 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:01,390 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:02,073 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/config.yaml -2024-04-30 13:44:02,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:44:02,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:44:02,285 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:44:03,133 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:44:03,135 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:44:03,135 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:44:03,135 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:44:03,136 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:44:03,136 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:44:03,136 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:44:03,137 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:44:04,073 INFO Thread-12 :161 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:44:04,443 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:44:04,445 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:44:04,445 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:44:04,445 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:44:04,445 DEBUG SenderThread:161 [sender.py:send():379] send: metric -2024-04-30 13:44:04,446 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:44:04,446 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:44:04,446 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:44:05,074 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:44:07,448 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:12,449 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:17,246 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:44:17,246 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:44:17,247 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:44:18,281 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:19,008 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:44:19,009 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:44:19,009 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:44:19,011 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:44:19,079 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:44:20,329 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:44:20,330 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:44:20,331 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:44:20,331 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:44:21,080 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:44:23,332 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:28,333 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:31,180 DEBUG SystemMonitor:161 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-30 13:44:31,182 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:44:32,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:44:32,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:44:32,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:44:34,335 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:34,874 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:44:34,875 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:44:34,875 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:44:34,878 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:44:35,085 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/config.yaml -2024-04-30 13:44:35,085 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:44:36,200 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:44:36,201 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:44:36,201 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:44:36,202 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:44:37,086 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:44:40,203 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:45,203 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:47,243 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:44:47,244 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:44:47,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:44:50,304 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:44:50,897 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:44:50,899 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:44:50,899 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:44:50,899 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:44:51,091 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:44:52,231 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:44:52,232 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:44:52,233 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:44:52,233 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:44:53,092 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:44:56,234 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:01,183 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:45:02,184 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:02,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:45:02,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:45:02,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:45:06,865 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:45:06,866 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:45:06,866 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:45:06,867 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:45:07,097 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:45:07,867 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:08,202 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:45:08,203 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:45:08,204 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:45:08,204 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:45:09,098 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:45:13,205 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:17,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:45:17,244 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:45:17,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:45:18,326 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:23,079 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:45:23,080 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:45:23,080 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:45:23,081 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:45:23,104 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:45:24,081 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:24,427 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:45:24,428 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:45:24,429 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:45:24,429 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:45:25,105 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:45:29,430 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:31,184 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:45:32,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:45:32,244 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:45:32,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:45:35,373 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:39,293 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:45:39,294 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:45:39,294 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:45:39,295 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:45:40,111 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:45:40,643 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:45:40,645 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:45:40,645 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:45:40,645 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:40,646 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:45:41,111 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:45:45,647 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:47,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:45:47,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:45:47,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:45:51,390 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:55,508 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:45:55,509 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:45:55,510 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:45:55,510 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:45:56,117 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:45:56,511 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:45:56,859 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:45:56,861 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:45:56,861 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:45:56,861 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:45:57,118 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:46:01,185 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:46:02,186 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:02,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:46:02,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:46:02,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:46:07,361 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:11,758 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:46:11,759 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:46:11,760 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:46:11,762 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:46:12,123 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:46:12,763 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:13,117 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:46:13,118 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:46:13,119 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:46:13,119 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:46:13,124 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:46:17,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:46:17,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:46:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:46:18,378 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:23,379 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:28,029 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:46:28,030 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:46:28,030 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:46:28,030 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:46:28,129 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:46:29,031 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:29,383 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:46:29,385 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:46:29,385 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:46:29,385 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:46:30,130 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:46:31,186 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:46:32,244 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:46:32,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:46:32,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:46:34,383 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:39,383 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:44,384 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:44,396 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:46:44,398 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:46:44,398 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:46:44,398 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:46:45,136 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:46:45,756 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:46:45,758 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:46:45,758 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:46:45,758 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:46:46,136 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:46:47,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:46:47,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:46:47,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:46:50,365 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:46:55,366 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:00,367 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:00,620 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:47:00,621 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:47:00,621 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:47:00,622 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:47:01,142 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:47:01,187 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:47:01,988 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:47:01,989 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:47:01,990 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:47:01,990 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:47:02,142 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:47:02,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:47:02,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:47:02,246 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:47:06,275 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:11,276 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:16,277 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:16,906 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:47:16,907 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:47:16,908 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:47:16,908 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:47:17,148 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:47:17,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:47:17,245 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:47:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:47:18,272 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:47:18,274 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:47:18,274 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:47:18,275 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:47:19,149 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:47:22,276 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:27,276 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:31,188 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:47:32,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:47:32,246 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:47:32,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:47:32,320 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:33,205 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:47:33,206 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:47:33,207 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:47:33,207 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:47:34,155 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:47:34,574 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:47:34,575 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:47:34,575 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:47:34,576 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:47:35,155 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:47:37,577 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:42,578 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:47,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:47:47,246 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:47:47,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:47:48,289 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:49,483 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:47:49,484 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:47:49,484 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:47:49,485 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:47:50,161 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:47:50,855 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:47:50,857 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:47:50,857 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:47:50,857 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:47:51,161 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:47:53,858 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:47:58,859 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:01,189 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:48:02,245 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:48:02,246 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:48:02,247 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:48:04,367 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:05,685 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:48:05,686 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:48:05,686 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:48:05,686 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:48:06,167 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:48:07,053 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:48:07,054 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:48:07,054 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:48:07,055 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:48:07,168 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:48:10,056 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:15,057 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:48:17,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:48:17,257 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:48:20,322 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:22,092 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:48:22,094 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:48:22,094 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:48:22,095 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:48:22,174 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:48:23,467 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:48:23,468 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:48:23,469 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:48:23,469 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:48:24,174 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:48:25,470 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:30,471 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:31,190 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:48:32,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:48:32,250 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:48:32,251 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:48:36,381 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:38,399 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:48:38,401 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:48:38,401 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:48:38,401 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:48:39,181 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:48:39,775 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:48:39,776 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:48:39,777 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:48:39,777 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:48:40,181 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:48:41,778 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:46,779 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:47,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:48:47,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:48:47,248 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:48:52,291 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:48:54,832 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:48:54,833 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:48:54,833 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:48:54,835 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:48:55,186 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:48:56,209 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:48:56,210 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:48:56,211 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:48:56,211 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:48:57,187 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:48:58,212 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:01,191 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:49:02,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:49:02,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:49:02,248 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:49:03,371 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:08,372 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:11,288 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:49:11,290 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:49:11,290 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:49:11,291 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:49:12,193 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:49:12,663 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:49:12,664 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:49:12,664 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:49:12,665 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:49:13,193 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:49:13,666 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:17,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:49:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:49:17,248 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:49:19,287 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:24,288 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:27,712 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:49:27,713 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:49:27,713 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:49:27,715 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:49:28,199 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:49:29,092 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:49:29,093 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:49:29,093 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:49:29,094 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:49:29,199 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:49:30,094 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:31,192 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:49:32,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:49:32,250 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:49:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:49:35,296 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:40,297 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:44,173 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:49:44,174 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:49:44,174 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:49:44,175 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:49:44,204 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:49:45,558 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:49:45,559 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:49:45,559 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:49:45,559 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:45,560 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:49:46,205 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:49:47,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:49:47,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:49:47,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:49:51,365 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:49:56,366 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:00,624 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:50:00,626 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:50:00,626 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:50:00,628 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:50:01,193 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:50:01,211 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:50:02,006 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:50:02,008 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:50:02,008 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:50:02,008 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:02,009 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:50:02,211 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:50:02,247 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:50:02,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:50:02,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:50:07,315 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:12,316 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:17,096 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:50:17,098 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:50:17,098 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:50:17,098 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:50:17,217 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:50:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:50:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:50:17,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:50:17,346 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:18,482 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:50:18,483 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:50:18,484 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:50:18,484 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:50:19,217 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:50:22,485 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:27,486 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:31,194 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:50:32,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:50:32,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:50:32,250 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:50:33,305 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:33,634 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:50:33,635 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:50:33,635 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:50:33,636 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:50:34,223 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:50:35,021 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:50:35,027 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:50:35,027 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:50:35,028 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:50:35,223 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:50:39,029 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:44,030 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:47,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:50:47,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:50:47,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:50:49,286 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:50,203 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:50:50,205 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:50:50,205 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:50:50,206 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:50:50,228 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:50:51,592 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:50:51,594 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:50:51,594 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:50:51,594 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:50:52,229 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:50:54,595 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:50:59,596 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:01,195 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:51:02,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:51:02,249 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:51:02,249 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:51:05,356 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:06,693 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:51:06,694 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:51:06,694 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:06,695 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:07,234 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:51:08,073 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:51:08,075 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:51:08,075 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:08,075 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:08,235 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:51:11,076 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:16,077 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:17,248 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:51:17,250 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:51:17,250 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:51:21,358 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:23,153 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:51:23,154 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:51:23,154 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:23,155 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:23,240 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:51:24,537 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:51:24,539 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:51:24,539 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:24,539 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:25,241 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:51:26,540 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:31,195 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:51:32,196 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:51:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:51:32,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:51:37,290 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:39,679 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:51:39,680 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:51:39,680 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:39,681 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:40,246 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:51:41,065 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:51:41,066 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:51:41,066 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:41,067 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:41,247 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:51:43,068 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:51:47,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:51:47,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:51:48,302 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:53,303 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:51:56,290 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:51:56,291 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:51:56,292 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:56,292 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:57,253 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:51:57,671 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:51:57,672 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:51:57,672 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:57,673 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:57,855 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: partial_history -2024-04-30 13:51:57,856 DEBUG SenderThread:161 [sender.py:send():379] send: telemetry -2024-04-30 13:51:57,856 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:57,856 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:57,857 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:57,857 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:57,858 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:57,858 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: pause -2024-04-30 13:51:57,858 INFO HandlerThread:161 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-04-30 13:51:57,858 INFO HandlerThread:161 [system_monitor.py:finish():203] Stopping system monitor -2024-04-30 13:51:57,859 DEBUG SystemMonitor:161 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-04-30 13:51:57,859 DEBUG SystemMonitor:161 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-04-30 13:51:57,860 INFO HandlerThread:161 [interfaces.py:finish():202] Joined cpu monitor -2024-04-30 13:51:57,860 INFO HandlerThread:161 [interfaces.py:finish():202] Joined disk monitor -2024-04-30 13:51:57,860 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:57,861 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:57,862 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:57,862 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:57,863 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:57,863 DEBUG SenderThread:161 [sender.py:send():379] send: history -2024-04-30 13:51:57,863 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: summary_record -2024-04-30 13:51:57,864 INFO SenderThread:161 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end -2024-04-30 13:51:57,874 INFO HandlerThread:161 [interfaces.py:finish():202] Joined gpu monitor -2024-04-30 13:51:57,874 INFO HandlerThread:161 [interfaces.py:finish():202] Joined memory monitor -2024-04-30 13:51:57,874 INFO HandlerThread:161 [interfaces.py:finish():202] Joined network monitor -2024-04-30 13:51:57,875 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:51:58,253 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/wandb-summary.json -2024-04-30 13:51:58,876 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:02,250 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:52:02,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:52:02,251 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:52:04,287 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:05,256 INFO Thread-12 :161 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240430_134330-p3yujlfg/files/config.yaml -2024-04-30 13:52:09,404 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:14,405 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:17,250 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:52:17,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:52:17,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:52:19,435 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:24,436 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:29,438 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:32,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:52:32,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:52:32,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:52:35,309 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:40,310 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:45,311 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:47,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:52:47,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:52:47,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:52:50,349 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:52:55,350 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:00,350 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:02,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:53:02,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:53:02,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:53:05,357 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:10,358 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:15,359 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:17,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:53:17,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:53:17,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:53:20,489 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:25,490 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:30,491 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:32,251 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:53:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:53:32,252 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:53:36,333 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:41,334 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:46,335 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:53:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:53:47,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:53:51,379 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:53:56,380 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:01,381 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:02,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:54:02,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:54:02,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:54:07,378 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:12,379 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:17,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:54:17,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:54:17,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:54:18,320 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:23,321 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:28,321 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:54:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:54:32,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:54:33,327 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:38,328 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:43,329 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:54:47,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:54:47,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:54:49,326 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:54,327 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:54:59,328 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:02,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:55:02,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:55:02,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:55:04,387 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:09,388 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:14,389 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:17,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:55:17,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:55:17,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:55:20,316 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:25,317 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:30,318 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:32,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:55:32,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:55:32,253 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:55:36,306 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:41,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:46,308 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:47,252 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:55:47,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:55:47,254 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:55:51,362 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:55:56,363 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:01,364 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:02,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:56:02,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:56:02,254 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:56:07,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:12,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:17,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:56:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:56:17,254 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:56:18,305 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:23,306 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:28,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:32,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:56:32,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:56:32,254 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:56:33,326 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:38,327 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:43,327 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:47,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:56:47,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:56:47,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:56:48,715 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:53,716 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:56:58,717 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:02,253 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:57:02,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:57:02,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:57:04,299 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:09,300 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:14,301 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:57:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:57:17,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:57:19,324 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:24,325 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:29,326 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:32,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:57:32,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:57:32,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:57:34,392 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:39,393 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:44,394 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:47,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:57:47,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:57:47,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:57:50,284 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:57:55,286 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:00,287 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:02,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:58:02,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:58:02,255 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:58:05,348 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:10,349 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:15,349 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:17,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:58:17,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:58:17,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:58:20,363 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:25,364 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:30,365 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:32,254 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:58:32,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:58:32,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:58:36,357 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:41,359 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:46,359 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:47,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:58:47,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:58:47,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:58:52,288 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:58:57,289 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:02,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:59:02,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:59:02,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:59:02,305 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:07,306 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:12,307 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:17,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:59:17,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:59:17,256 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:59:17,379 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:22,380 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:27,381 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:32,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:59:32,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:59:32,257 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:59:32,414 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: resume -2024-04-30 13:59:32,414 INFO HandlerThread:161 [handler.py:handle_request_resume():699] starting system metrics thread -2024-04-30 13:59:32,414 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor -2024-04-30 13:59:32,414 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-30 13:59:32,415 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring -2024-04-30 13:59:32,415 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring -2024-04-30 13:59:32,416 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring -2024-04-30 13:59:32,418 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring -2024-04-30 13:59:32,420 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring -2024-04-30 13:59:32,454 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: pause -2024-04-30 13:59:32,454 INFO HandlerThread:161 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-04-30 13:59:32,454 INFO HandlerThread:161 [system_monitor.py:finish():203] Stopping system monitor -2024-04-30 13:59:32,455 DEBUG SystemMonitor:161 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-30 13:59:32,455 DEBUG SystemMonitor:161 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-04-30 13:59:32,455 DEBUG SystemMonitor:161 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-04-30 13:59:32,455 INFO HandlerThread:161 [interfaces.py:finish():202] Joined cpu monitor -2024-04-30 13:59:32,456 INFO HandlerThread:161 [interfaces.py:finish():202] Joined disk monitor -2024-04-30 13:59:32,466 INFO HandlerThread:161 [interfaces.py:finish():202] Joined gpu monitor -2024-04-30 13:59:32,466 INFO HandlerThread:161 [interfaces.py:finish():202] Joined memory monitor -2024-04-30 13:59:32,467 INFO HandlerThread:161 [interfaces.py:finish():202] Joined network monitor -2024-04-30 13:59:32,467 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:59:32,467 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:33,550 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: resume -2024-04-30 13:59:33,550 INFO HandlerThread:161 [handler.py:handle_request_resume():699] starting system metrics thread -2024-04-30 13:59:33,550 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor -2024-04-30 13:59:33,550 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-30 13:59:33,551 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring -2024-04-30 13:59:33,551 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring -2024-04-30 13:59:33,552 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring -2024-04-30 13:59:33,553 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring -2024-04-30 13:59:33,554 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring -2024-04-30 13:59:35,391 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: pause -2024-04-30 13:59:35,392 INFO HandlerThread:161 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-04-30 13:59:35,392 INFO HandlerThread:161 [system_monitor.py:finish():203] Stopping system monitor -2024-04-30 13:59:35,392 INFO HandlerThread:161 [interfaces.py:finish():202] Joined cpu monitor -2024-04-30 13:59:35,393 DEBUG SystemMonitor:161 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-30 13:59:35,393 DEBUG SystemMonitor:161 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-04-30 13:59:35,393 INFO HandlerThread:161 [interfaces.py:finish():202] Joined disk monitor -2024-04-30 13:59:35,393 DEBUG SystemMonitor:161 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-04-30 13:59:35,405 INFO HandlerThread:161 [interfaces.py:finish():202] Joined gpu monitor -2024-04-30 13:59:35,406 INFO HandlerThread:161 [interfaces.py:finish():202] Joined memory monitor -2024-04-30 13:59:35,406 INFO HandlerThread:161 [interfaces.py:finish():202] Joined network monitor -2024-04-30 13:59:35,406 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 13:59:38,407 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:43,408 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:47,255 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 13:59:47,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 13:59:47,257 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 13:59:49,351 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:54,351 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 13:59:59,352 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 14:00:02,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 14:00:02,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 14:00:02,257 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 14:00:05,312 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: resume -2024-04-30 14:00:05,312 INFO HandlerThread:161 [handler.py:handle_request_resume():699] starting system metrics thread -2024-04-30 14:00:05,312 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor -2024-04-30 14:00:05,313 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-30 14:00:05,313 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring -2024-04-30 14:00:05,314 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring -2024-04-30 14:00:05,314 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring -2024-04-30 14:00:05,315 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring -2024-04-30 14:00:05,316 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring -2024-04-30 14:00:05,334 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 14:00:05,340 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: pause -2024-04-30 14:00:05,340 INFO HandlerThread:161 [handler.py:handle_request_pause():708] stopping system metrics thread -2024-04-30 14:00:05,340 INFO HandlerThread:161 [system_monitor.py:finish():203] Stopping system monitor -2024-04-30 14:00:05,340 DEBUG SystemMonitor:161 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-04-30 14:00:05,340 DEBUG SystemMonitor:161 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-04-30 14:00:05,340 DEBUG SystemMonitor:161 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-04-30 14:00:05,341 INFO HandlerThread:161 [interfaces.py:finish():202] Joined cpu monitor -2024-04-30 14:00:05,344 INFO HandlerThread:161 [interfaces.py:finish():202] Joined disk monitor -2024-04-30 14:00:05,353 INFO HandlerThread:161 [interfaces.py:finish():202] Joined gpu monitor -2024-04-30 14:00:05,353 INFO HandlerThread:161 [interfaces.py:finish():202] Joined memory monitor -2024-04-30 14:00:05,353 INFO HandlerThread:161 [interfaces.py:finish():202] Joined network monitor -2024-04-30 14:00:05,354 DEBUG SenderThread:161 [sender.py:send():379] send: stats -2024-04-30 14:00:10,355 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 14:00:15,356 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 14:00:17,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: internal_messages -2024-04-30 14:00:17,256 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: stop_status -2024-04-30 14:00:17,257 DEBUG SenderThread:161 [sender.py:send_request():406] send_request: stop_status -2024-04-30 14:00:20,361 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report -2024-04-30 14:00:22,024 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: resume -2024-04-30 14:00:22,024 INFO HandlerThread:161 [handler.py:handle_request_resume():699] starting system metrics thread -2024-04-30 14:00:22,024 INFO HandlerThread:161 [system_monitor.py:start():194] Starting system monitor -2024-04-30 14:00:22,024 INFO SystemMonitor:161 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-04-30 14:00:22,025 INFO SystemMonitor:161 [interfaces.py:start():190] Started cpu monitoring -2024-04-30 14:00:22,025 INFO SystemMonitor:161 [interfaces.py:start():190] Started disk monitoring -2024-04-30 14:00:22,027 INFO SystemMonitor:161 [interfaces.py:start():190] Started gpu monitoring -2024-04-30 14:00:22,027 INFO SystemMonitor:161 [interfaces.py:start():190] Started memory monitoring -2024-04-30 14:00:22,028 INFO SystemMonitor:161 [interfaces.py:start():190] Started network monitoring -2024-04-30 14:00:25,362 DEBUG HandlerThread:161 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:22,360 DEBUG HandlerThread:138 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-01 08:36:22,361 INFO HandlerThread:138 [system_monitor.py:probe():229] Finished publishing system info +2024-05-01 08:36:22,368 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:22,368 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: keepalive +2024-05-01 08:36:22,369 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:22,369 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: keepalive +2024-05-01 08:36:22,369 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:22,369 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: keepalive +2024-05-01 08:36:22,369 DEBUG SenderThread:138 [sender.py:send():379] send: files +2024-05-01 08:36:22,370 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-metadata.json with policy now +2024-05-01 08:36:22,679 INFO wandb-upload_0:138 [upload_job.py:push():131] Uploaded file /tmp/tmp3psgvkztwandb/prksz0t1-wandb-metadata.json +2024-05-01 08:36:23,203 INFO Thread-12 :138 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-metadata.json +2024-05-01 08:36:23,367 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: python_packages +2024-05-01 08:36:23,368 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: python_packages +2024-05-01 08:36:23,371 DEBUG SenderThread:138 [sender.py:send():379] send: telemetry +2024-05-01 08:36:23,383 DEBUG SenderThread:138 [sender.py:send():379] send: config +2024-05-01 08:36:23,384 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:36:23,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:36:23,386 DEBUG SenderThread:138 [sender.py:send():379] send: telemetry +2024-05-01 08:36:23,386 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:36:23,386 WARNING SenderThread:138 [sender.py:send_metric():1341] Seen metric with glob (shouldn't happen) +2024-05-01 08:36:23,387 DEBUG SenderThread:138 [sender.py:send():379] send: telemetry +2024-05-01 08:36:23,387 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:36:23,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:36:24,203 INFO Thread-12 :138 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/requirements.txt +2024-05-01 08:36:27,545 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:32,545 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:37,551 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:38,208 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/config.yaml +2024-05-01 08:36:38,368 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:36:38,369 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:36:38,409 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:36:43,460 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:48,461 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:53,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:36:53,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:36:53,373 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:36:54,447 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:36:57,961 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:36:57,964 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:36:57,964 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:36:57,964 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:36:57,964 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:36:57,964 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:36:57,964 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:36:57,966 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:36:58,215 INFO Thread-12 :138 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:36:59,967 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:00,913 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:37:00,916 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:37:00,916 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:37:00,916 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:37:00,916 DEBUG SenderThread:138 [sender.py:send():379] send: metric +2024-05-01 08:37:00,917 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:37:00,917 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:37:00,919 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:37:01,217 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:37:05,920 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:07,326 DEBUG SystemMonitor:138 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-01 08:37:07,327 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:37:08,370 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:37:08,370 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:37:08,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:37:11,441 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:12,221 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/config.yaml +2024-05-01 08:37:16,596 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:21,597 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:23,369 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:37:23,369 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:37:23,410 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:37:27,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:32,535 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:35,717 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:37:35,718 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:37:35,718 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:37:35,721 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:37:36,230 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:37:37,329 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:37:38,330 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:38,369 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:37:38,369 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:37:38,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:37:38,783 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:37:38,785 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:37:38,785 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:37:38,785 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:37:39,231 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:37:43,786 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:48,787 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:53,369 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:37:53,370 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:37:53,370 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:37:54,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:37:59,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:04,535 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:07,330 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:38:08,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:38:08,372 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:38:08,412 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:38:09,538 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:14,539 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:14,639 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:38:14,640 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:38:14,640 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:38:14,641 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:38:15,245 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:38:17,672 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:38:17,674 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:38:17,674 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:38:17,676 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:38:18,246 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:38:19,677 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:23,370 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:38:23,370 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:38:23,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:38:25,544 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:30,545 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:35,546 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:37,331 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:38:38,370 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:38:38,370 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:38:38,411 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:38:41,437 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:46,437 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:51,438 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:38:52,850 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:38:52,851 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:38:52,852 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:38:52,853 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:38:53,259 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:38:53,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:38:53,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:38:53,375 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:38:55,897 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:38:55,898 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:38:55,898 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:38:55,898 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:38:56,260 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:38:56,899 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:01,900 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:06,901 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:07,331 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:39:08,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:39:08,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:39:08,373 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:39:12,458 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:17,459 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:22,460 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:23,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:39:23,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:39:23,372 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:39:27,472 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:31,256 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:39:31,257 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:39:31,257 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:39:31,260 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:39:31,274 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:39:33,261 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:34,301 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:39:34,302 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:39:34,302 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:39:34,303 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:39:35,275 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:39:37,333 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:39:38,334 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:38,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:39:38,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:39:38,372 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:39:43,447 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:48,448 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:53,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:39:53,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:39:53,373 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:39:54,442 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:39:59,443 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:04,444 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:07,333 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:40:08,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:40:08,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:40:08,373 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:40:09,500 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:09,681 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:40:09,682 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:40:09,682 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:40:09,682 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:40:10,289 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:40:12,729 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:40:12,730 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:40:12,730 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:40:12,731 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:40:13,290 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:40:14,732 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:19,733 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:23,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:40:23,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:40:23,373 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:40:25,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:30,535 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:35,536 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:37,334 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:40:38,371 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:40:38,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:40:38,373 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:40:41,443 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:46,444 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:47,736 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:40:47,737 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:40:47,738 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:40:47,740 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:40:48,302 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:40:50,791 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:40:50,793 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:40:50,793 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:40:50,794 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:40:51,303 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:40:51,794 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:40:53,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:40:53,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:40:53,373 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:40:57,499 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:02,500 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:07,336 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:41:08,336 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:08,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:41:08,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:41:08,374 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:41:13,475 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:18,476 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:23,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:41:23,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:41:23,374 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:41:23,541 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:25,919 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:41:25,920 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:41:25,920 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:41:25,920 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:41:26,317 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:41:28,921 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:28,962 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:41:28,963 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:41:28,963 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:41:28,964 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:41:29,318 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:41:33,965 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:37,337 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:41:38,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:41:38,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:41:38,374 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:41:39,518 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:44,519 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:49,520 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:41:53,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:41:53,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:41:53,374 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:41:55,438 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:00,439 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:04,022 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:42:04,023 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:42:04,024 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:42:04,024 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:42:04,332 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:42:06,025 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:07,067 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:42:07,068 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:42:07,068 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:42:07,069 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:42:07,333 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:42:07,338 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:42:08,372 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:42:08,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:42:08,374 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:42:11,441 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:16,442 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:21,443 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:23,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:42:23,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:42:23,375 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:42:26,497 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:31,498 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:36,499 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:37,339 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:42:38,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:42:38,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:42:38,375 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:42:41,767 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:42:41,769 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:42:41,769 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:42:41,769 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:41,770 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:42:42,347 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:42:44,814 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:42:44,815 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:42:44,815 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:42:44,816 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:42:45,348 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:42:46,816 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:51,817 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:42:53,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:42:53,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:42:53,374 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:42:57,510 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:02,511 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:07,340 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:43:08,341 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:08,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:43:08,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:43:08,374 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:43:13,536 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:18,537 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:19,564 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:43:19,565 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:43:19,566 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:43:19,566 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:43:20,361 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:43:22,594 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:43:22,595 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:43:22,596 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:43:22,596 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:43:23,363 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:43:23,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:43:23,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:43:23,374 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:43:24,469 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:29,470 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:34,471 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:37,341 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:43:38,373 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:43:38,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:43:38,374 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:43:40,457 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:45,458 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:50,459 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:53,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:43:53,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:43:53,375 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:43:55,541 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:43:57,518 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:43:57,519 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:43:57,519 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:43:57,519 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:43:58,380 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:44:00,556 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:44:00,557 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:44:00,558 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:44:00,558 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:00,558 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:44:01,381 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:44:05,559 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:07,342 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:44:08,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:44:08,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:44:08,376 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:44:11,513 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:16,514 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:21,515 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:23,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:44:23,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:44:23,375 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:44:26,529 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:31,530 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:35,061 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:44:35,063 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:44:35,063 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:44:35,063 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:44:35,393 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:44:37,064 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:37,343 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:44:38,101 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:44:38,103 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:44:38,103 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:44:38,103 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:44:38,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:44:38,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:44:38,375 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:44:38,395 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:44:42,496 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:47,497 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:52,497 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:44:53,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:44:53,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:44:53,375 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:44:58,443 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:03,444 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:07,344 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:45:08,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:45:08,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:45:08,375 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:45:08,508 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:13,011 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:45:13,013 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:45:13,013 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:45:13,014 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:45:13,408 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:45:14,014 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:16,054 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:45:16,055 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:45:16,055 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:45:16,055 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:45:16,409 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:45:19,056 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:23,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:45:23,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:45:23,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:45:24,452 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:29,453 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:34,453 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:37,345 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:45:38,374 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:45:38,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:45:38,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:45:39,492 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:44,493 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:49,495 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:45:51,018 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:45:51,019 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:45:51,020 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:45:51,020 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:45:51,422 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:45:53,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:45:53,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:45:53,376 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:45:54,061 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:45:54,063 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:45:54,063 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:45:54,064 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:45:54,424 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:45:55,064 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:00,066 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:05,067 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:07,350 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:46:08,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:46:08,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:46:08,376 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:46:10,511 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:15,512 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:20,513 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:23,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:46:23,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:46:23,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:46:25,518 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:28,948 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:46:28,949 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:46:28,949 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:46:28,950 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:46:29,436 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:46:30,951 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:31,981 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:46:31,982 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:46:31,982 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:46:31,982 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:46:32,438 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:46:35,983 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:37,347 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:46:38,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:46:38,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:46:38,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:46:41,502 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:46,503 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:51,504 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:46:53,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:46:53,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:46:53,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:46:56,537 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:01,538 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:06,539 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:06,789 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:47:06,791 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:47:06,791 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:47:06,793 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:47:07,347 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:47:07,451 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:47:08,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:47:08,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:47:08,376 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:47:09,827 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:47:09,828 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:47:09,828 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:47:09,829 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:47:10,452 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:47:11,830 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:16,831 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:21,832 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:23,375 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:47:23,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:47:23,376 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:47:27,511 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:32,512 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:37,348 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:47:38,349 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:38,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:47:38,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:47:38,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:47:43,523 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:44,849 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:47:44,850 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:47:44,851 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:47:44,851 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:47:45,464 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:47:47,887 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:47:47,888 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:47:47,889 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:47:47,889 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:47:48,465 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:47:48,890 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:53,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:47:53,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:47:53,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:47:54,460 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:47:59,461 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:04,462 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:07,349 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:48:08,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:48:08,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:48:08,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:48:10,433 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:15,433 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:20,434 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:22,886 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:48:22,887 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:48:22,888 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:48:22,888 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:48:23,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:48:23,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:48:23,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:48:23,477 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:48:25,460 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:25,927 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:48:25,928 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:48:25,929 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:48:25,929 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:48:26,478 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:48:30,931 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:35,931 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:37,350 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:48:38,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:48:38,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:48:38,377 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:48:41,511 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:46,511 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:51,512 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:48:53,376 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:48:53,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:48:53,378 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:48:57,465 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:00,608 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:49:00,609 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:49:00,609 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:49:00,609 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:49:01,491 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:49:02,610 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:03,638 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:49:03,639 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:49:03,640 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:49:03,640 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:49:04,492 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:49:07,351 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:49:08,352 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:08,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:49:08,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:49:08,378 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:49:13,456 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:18,457 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:23,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:49:23,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:49:23,378 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:49:24,455 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:29,457 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:34,458 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:37,352 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:49:38,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:49:38,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:49:38,379 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:49:38,539 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:49:38,540 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:49:38,540 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:49:38,541 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:49:39,504 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:49:39,542 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:41,577 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:49:41,578 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:49:41,579 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:49:41,579 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:49:42,505 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:49:44,580 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:49,581 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:49:53,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:49:53,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:49:53,378 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:49:55,453 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:00,454 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:05,455 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:07,354 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:50:08,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:50:08,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:50:08,379 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:50:10,488 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:15,489 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:16,466 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:50:16,467 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:50:16,467 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:50:16,467 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:50:16,518 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:50:19,502 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:50:19,503 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:50:19,503 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:50:19,504 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:50:19,519 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:50:20,504 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:23,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:50:23,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:50:23,378 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:50:26,464 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:31,465 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:36,466 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:37,355 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:50:38,377 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:50:38,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:50:38,379 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:50:41,491 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:46,492 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:51,493 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:53,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:50:53,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:50:53,379 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:50:54,298 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:50:54,299 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:50:54,299 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:50:54,300 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:50:54,532 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:50:57,301 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:50:57,329 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:50:57,330 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:50:57,330 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:50:57,331 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:50:57,533 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:51:02,332 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:07,333 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:07,355 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:51:08,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:51:08,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:51:08,379 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:51:12,449 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:17,450 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:22,451 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:23,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:51:23,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:51:23,379 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:51:27,538 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:32,300 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:51:32,301 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:51:32,301 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:51:32,302 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:51:32,546 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:51:33,302 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:35,335 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:51:35,337 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:51:35,337 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:51:35,338 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:51:35,547 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:51:37,356 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:51:38,357 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:38,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:51:38,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:51:38,379 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:51:43,550 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:48,551 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:53,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:51:53,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:51:53,380 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:51:54,528 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:51:59,529 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:04,530 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:07,357 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:52:08,378 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:52:08,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:52:08,379 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:52:10,009 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:52:10,010 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:52:10,010 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:52:10,011 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:10,011 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:52:10,561 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:52:13,047 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:52:13,049 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:52:13,049 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:52:13,049 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:52:13,562 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:52:15,050 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:20,051 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:23,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:52:23,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:52:23,380 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:52:25,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:30,535 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:35,536 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:37,358 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:52:38,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:52:38,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:52:38,380 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:52:40,536 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:45,537 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:48,077 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:52:48,079 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:52:48,079 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:52:48,079 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:52:48,575 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:52:51,080 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:52:51,113 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:52:51,114 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:52:51,115 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:52:51,115 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:52:51,576 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:52:53,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:52:53,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:52:53,381 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:52:56,496 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:01,497 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:06,497 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:07,360 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:53:08,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:53:08,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:53:08,380 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:53:12,483 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:17,483 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:22,484 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:23,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:53:23,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:53:23,380 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:53:25,972 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:53:25,973 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:53:25,973 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:25,973 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:26,588 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:53:27,974 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:29,000 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:53:29,001 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:53:29,001 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:29,002 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:29,590 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:53:33,003 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:37,361 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:53:38,361 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:38,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:53:38,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:53:38,380 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:53:43,492 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:48,493 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:53,379 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:53:53,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:53:53,381 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:53:53,523 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:53:53,718 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:53:53,719 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:53:53,719 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:53,720 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:54,599 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:53:56,756 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:53:56,758 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:53:56,758 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:56,758 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:57,011 DEBUG SenderThread:138 [sender.py:send():379] send: telemetry +2024-05-01 08:53:57,012 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:57,014 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:57,015 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:57,015 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: partial_history +2024-05-01 08:53:57,017 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: pause +2024-05-01 08:53:57,017 INFO HandlerThread:138 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-01 08:53:57,017 INFO HandlerThread:138 [system_monitor.py:finish():203] Stopping system monitor +2024-05-01 08:53:57,017 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:57,017 DEBUG SystemMonitor:138 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-01 08:53:57,018 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:57,018 INFO HandlerThread:138 [interfaces.py:finish():202] Joined cpu monitor +2024-05-01 08:53:57,018 DEBUG SystemMonitor:138 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-01 08:53:57,019 INFO HandlerThread:138 [interfaces.py:finish():202] Joined disk monitor +2024-05-01 08:53:57,021 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:57,021 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:57,021 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:57,022 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:57,022 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:57,023 DEBUG SenderThread:138 [sender.py:send():379] send: history +2024-05-01 08:53:57,023 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: summary_record +2024-05-01 08:53:57,023 INFO SenderThread:138 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-01 08:53:57,034 INFO HandlerThread:138 [interfaces.py:finish():202] Joined gpu monitor +2024-05-01 08:53:57,035 INFO HandlerThread:138 [interfaces.py:finish():202] Joined memory monitor +2024-05-01 08:53:57,035 INFO HandlerThread:138 [interfaces.py:finish():202] Joined network monitor +2024-05-01 08:53:57,035 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 08:53:57,601 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/wandb-summary.json +2024-05-01 08:53:59,036 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:04,037 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:08,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:54:08,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:54:08,381 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:54:09,485 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:14,486 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:19,487 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:23,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:54:23,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:54:23,381 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:54:24,546 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:25,611 INFO Thread-12 :138 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240501_083606-td31zvzm/files/config.yaml +2024-05-01 08:54:29,679 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:34,680 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:38,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:54:38,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:54:38,381 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:54:40,532 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:45,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:50,535 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:54:53,380 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:54:53,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:54:53,381 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:54:56,471 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:01,472 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:06,473 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:08,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:55:08,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:55:08,382 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:55:11,509 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:16,510 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:21,511 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:23,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:55:23,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:55:23,382 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:55:26,536 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:31,537 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:36,538 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:38,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:55:38,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:55:38,382 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:55:42,504 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:47,505 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:52,506 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:55:53,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:55:53,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:55:53,382 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:55:57,514 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:02,515 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:07,516 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:08,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:56:08,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:56:08,382 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:56:12,530 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:17,531 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:22,532 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:23,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:56:23,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:56:23,383 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:56:27,552 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:32,553 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:37,555 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:38,381 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:56:38,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:56:38,382 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:56:43,523 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:48,524 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:53,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:56:53,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:56:53,383 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:56:54,470 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:56:59,472 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:04,473 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:08,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:57:08,383 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:57:08,383 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:57:10,452 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:15,453 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:20,455 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:23,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:57:23,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:57:23,383 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:57:25,532 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:30,533 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:35,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:38,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:57:38,383 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:57:38,383 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:57:41,464 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:46,465 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:51,467 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:57:53,382 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:57:53,383 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:57:53,384 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:57:56,496 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:01,497 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:06,498 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:08,383 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:58:08,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:58:08,384 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:58:11,533 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:16,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:21,535 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:23,383 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:58:23,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:58:23,384 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:58:27,494 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:32,495 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:37,496 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:38,383 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:58:38,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:58:38,385 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:58:42,498 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:47,499 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:52,500 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:58:53,383 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:58:53,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:58:53,384 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:58:58,482 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:03,483 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:08,383 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:59:08,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:59:08,384 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:59:08,500 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:13,501 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:18,502 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:23,383 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:59:23,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:59:23,385 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:59:23,541 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:28,542 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:33,543 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:38,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:59:38,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:59:38,385 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:59:38,543 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:43,545 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:48,545 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:53,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 08:59:53,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 08:59:53,386 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 08:59:54,512 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 08:59:59,514 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:04,515 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:08,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:00:08,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:00:08,385 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:00:10,497 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:15,498 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:20,499 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:23,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:00:23,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:00:23,385 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:00:26,486 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:31,487 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:36,488 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:38,384 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:00:38,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:00:38,385 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:00:41,543 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:46,545 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:51,546 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:00:53,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:00:53,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:00:53,386 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:00:57,445 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:02,446 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:07,447 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:08,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:01:08,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:01:08,386 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:01:12,554 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:17,554 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:22,556 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:23,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:01:23,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:01:23,386 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:01:28,488 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:33,489 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:38,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:01:38,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:01:38,386 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:01:39,468 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:44,470 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:49,471 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:53,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:01:53,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:01:53,386 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:01:54,505 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:01:59,507 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:04,508 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:08,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:02:08,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:02:08,386 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:02:10,502 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:15,503 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:20,504 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:23,385 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:02:23,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:02:23,387 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:02:25,520 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:30,521 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:35,522 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:38,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:02:38,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:02:38,387 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:02:40,527 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:45,528 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:50,529 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:02:53,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:02:53,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:02:53,387 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:02:56,503 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:01,504 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:06,505 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:08,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:03:08,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:03:08,387 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:03:11,540 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:16,542 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:21,542 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:23,386 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:03:23,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:03:23,387 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:03:26,556 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:31,557 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:36,558 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:38,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:03:38,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:03:38,388 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:03:42,542 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:47,543 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:52,545 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:03:53,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:03:53,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:03:53,388 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:03:58,497 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:03,498 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:08,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:04:08,388 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:04:08,388 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:04:08,566 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:13,568 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:18,568 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:23,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:04:23,388 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:04:23,389 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:04:24,487 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:29,488 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:34,489 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:38,387 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:04:38,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:04:38,389 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:04:39,540 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:44,541 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:49,542 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:04:53,388 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:04:53,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:04:53,389 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:04:55,533 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:00,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:05,535 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:08,388 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:05:08,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:05:08,389 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:05:10,536 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:15,537 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:20,538 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:23,388 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:05:23,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:05:23,389 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:05:25,543 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:30,544 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:35,545 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:38,388 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:05:38,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:05:38,389 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:05:41,458 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:46,459 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:51,460 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:05:53,388 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:05:53,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:05:53,390 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:05:56,555 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:01,556 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:06,557 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:08,388 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:06:08,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:06:08,390 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:06:12,476 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:17,477 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:22,478 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:23,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:06:23,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:06:23,390 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:06:27,528 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:32,529 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:37,530 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:38,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:06:38,390 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:06:38,390 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:06:43,516 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:48,517 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:53,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:06:53,390 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:06:53,391 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:06:53,521 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:06:58,522 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:03,523 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:08,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:07:08,390 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:07:08,390 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:07:09,515 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:14,516 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:19,517 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:23,389 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:07:23,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:07:23,391 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:07:25,448 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:30,449 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:35,450 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:38,390 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:07:38,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:07:38,391 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:07:40,493 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:45,494 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:50,496 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:07:53,390 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:07:53,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:07:53,391 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:07:56,485 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:01,486 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:06,487 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:08,390 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:08:08,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:08:08,391 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:08:11,541 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:16,543 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:21,543 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:23,390 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:08:23,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:08:23,392 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:08:27,459 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:32,461 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:37,462 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:38,390 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:08:38,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:08:38,392 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:08:42,534 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:47,535 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:52,537 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:08:53,390 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:08:53,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:08:53,392 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:08:55,434 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: resume +2024-05-01 09:08:55,434 INFO HandlerThread:138 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-01 09:08:55,434 INFO HandlerThread:138 [system_monitor.py:start():194] Starting system monitor +2024-05-01 09:08:55,435 INFO SystemMonitor:138 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-01 09:08:55,435 INFO SystemMonitor:138 [interfaces.py:start():190] Started cpu monitoring +2024-05-01 09:08:55,436 INFO SystemMonitor:138 [interfaces.py:start():190] Started disk monitoring +2024-05-01 09:08:55,438 INFO SystemMonitor:138 [interfaces.py:start():190] Started gpu monitoring +2024-05-01 09:08:55,439 INFO SystemMonitor:138 [interfaces.py:start():190] Started memory monitoring +2024-05-01 09:08:55,440 INFO SystemMonitor:138 [interfaces.py:start():190] Started network monitoring +2024-05-01 09:08:55,461 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: pause +2024-05-01 09:08:55,461 INFO HandlerThread:138 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-01 09:08:55,461 INFO HandlerThread:138 [system_monitor.py:finish():203] Stopping system monitor +2024-05-01 09:08:55,461 INFO HandlerThread:138 [interfaces.py:finish():202] Joined cpu monitor +2024-05-01 09:08:55,462 INFO HandlerThread:138 [interfaces.py:finish():202] Joined disk monitor +2024-05-01 09:08:55,462 DEBUG SystemMonitor:138 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-01 09:08:55,462 DEBUG SystemMonitor:138 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-01 09:08:55,463 DEBUG SystemMonitor:138 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-01 09:08:55,474 INFO HandlerThread:138 [interfaces.py:finish():202] Joined gpu monitor +2024-05-01 09:08:55,474 INFO HandlerThread:138 [interfaces.py:finish():202] Joined memory monitor +2024-05-01 09:08:55,475 INFO HandlerThread:138 [interfaces.py:finish():202] Joined network monitor +2024-05-01 09:08:55,475 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 09:08:58,476 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:09:03,477 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:09:08,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:09:08,392 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:09:08,392 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:09:08,516 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:09:13,517 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:09:18,518 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:09:23,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:09:23,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:09:23,392 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:09:23,537 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:09:28,538 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:09:29,001 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: resume +2024-05-01 09:09:29,001 INFO HandlerThread:138 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-01 09:09:29,001 INFO HandlerThread:138 [system_monitor.py:start():194] Starting system monitor +2024-05-01 09:09:29,001 INFO SystemMonitor:138 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-01 09:09:29,002 INFO SystemMonitor:138 [interfaces.py:start():190] Started cpu monitoring +2024-05-01 09:09:29,003 INFO SystemMonitor:138 [interfaces.py:start():190] Started disk monitoring +2024-05-01 09:09:29,004 INFO SystemMonitor:138 [interfaces.py:start():190] Started gpu monitoring +2024-05-01 09:09:29,005 INFO SystemMonitor:138 [interfaces.py:start():190] Started memory monitoring +2024-05-01 09:09:29,006 INFO SystemMonitor:138 [interfaces.py:start():190] Started network monitoring +2024-05-01 09:09:29,042 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: pause +2024-05-01 09:09:29,042 INFO HandlerThread:138 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-01 09:09:29,042 INFO HandlerThread:138 [system_monitor.py:finish():203] Stopping system monitor +2024-05-01 09:09:29,042 DEBUG SystemMonitor:138 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-01 09:09:29,042 INFO HandlerThread:138 [interfaces.py:finish():202] Joined cpu monitor +2024-05-01 09:09:29,042 DEBUG SystemMonitor:138 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-01 09:09:29,043 INFO HandlerThread:138 [interfaces.py:finish():202] Joined disk monitor +2024-05-01 09:09:29,043 DEBUG SystemMonitor:138 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-01 09:09:29,054 INFO HandlerThread:138 [interfaces.py:finish():202] Joined gpu monitor +2024-05-01 09:09:29,054 INFO HandlerThread:138 [interfaces.py:finish():202] Joined memory monitor +2024-05-01 09:09:29,054 INFO HandlerThread:138 [interfaces.py:finish():202] Joined network monitor +2024-05-01 09:09:29,055 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 09:09:32,587 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: resume +2024-05-01 09:09:32,587 INFO HandlerThread:138 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-01 09:09:32,587 INFO HandlerThread:138 [system_monitor.py:start():194] Starting system monitor +2024-05-01 09:09:32,587 INFO SystemMonitor:138 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-01 09:09:32,587 INFO SystemMonitor:138 [interfaces.py:start():190] Started cpu monitoring +2024-05-01 09:09:32,588 INFO SystemMonitor:138 [interfaces.py:start():190] Started disk monitoring +2024-05-01 09:09:32,591 INFO SystemMonitor:138 [interfaces.py:start():190] Started gpu monitoring +2024-05-01 09:09:32,592 INFO SystemMonitor:138 [interfaces.py:start():190] Started memory monitoring +2024-05-01 09:09:32,594 INFO SystemMonitor:138 [interfaces.py:start():190] Started network monitoring +2024-05-01 09:09:33,733 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: pause +2024-05-01 09:09:33,733 INFO HandlerThread:138 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-01 09:09:33,733 INFO HandlerThread:138 [system_monitor.py:finish():203] Stopping system monitor +2024-05-01 09:09:33,734 DEBUG SystemMonitor:138 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-01 09:09:33,734 DEBUG SystemMonitor:138 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-01 09:09:33,734 DEBUG SystemMonitor:138 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-01 09:09:33,736 INFO HandlerThread:138 [interfaces.py:finish():202] Joined cpu monitor +2024-05-01 09:09:33,737 INFO HandlerThread:138 [interfaces.py:finish():202] Joined disk monitor +2024-05-01 09:09:33,748 INFO HandlerThread:138 [interfaces.py:finish():202] Joined gpu monitor +2024-05-01 09:09:33,748 INFO HandlerThread:138 [interfaces.py:finish():202] Joined memory monitor +2024-05-01 09:09:33,749 INFO HandlerThread:138 [interfaces.py:finish():202] Joined network monitor +2024-05-01 09:09:33,749 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 09:09:33,750 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:09:38,391 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-01 09:09:38,392 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: stop_status +2024-05-01 09:09:38,393 DEBUG SenderThread:138 [sender.py:send_request():406] send_request: stop_status +2024-05-01 09:09:39,469 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report +2024-05-01 09:09:42,797 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: resume +2024-05-01 09:09:42,797 INFO HandlerThread:138 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-01 09:09:42,797 INFO HandlerThread:138 [system_monitor.py:start():194] Starting system monitor +2024-05-01 09:09:42,798 INFO SystemMonitor:138 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-01 09:09:42,798 INFO SystemMonitor:138 [interfaces.py:start():190] Started cpu monitoring +2024-05-01 09:09:42,799 INFO SystemMonitor:138 [interfaces.py:start():190] Started disk monitoring +2024-05-01 09:09:42,800 INFO SystemMonitor:138 [interfaces.py:start():190] Started gpu monitoring +2024-05-01 09:09:42,801 INFO SystemMonitor:138 [interfaces.py:start():190] Started memory monitoring +2024-05-01 09:09:42,803 INFO SystemMonitor:138 [interfaces.py:start():190] Started network monitoring +2024-05-01 09:09:42,837 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: pause +2024-05-01 09:09:42,838 INFO HandlerThread:138 [handler.py:handle_request_pause():708] stopping system metrics thread +2024-05-01 09:09:42,838 INFO HandlerThread:138 [system_monitor.py:finish():203] Stopping system monitor +2024-05-01 09:09:42,838 INFO HandlerThread:138 [interfaces.py:finish():202] Joined cpu monitor +2024-05-01 09:09:42,838 INFO HandlerThread:138 [interfaces.py:finish():202] Joined disk monitor +2024-05-01 09:09:42,839 DEBUG SystemMonitor:138 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-01 09:09:42,839 DEBUG SystemMonitor:138 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-01 09:09:42,839 DEBUG SystemMonitor:138 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-01 09:09:42,849 INFO HandlerThread:138 [interfaces.py:finish():202] Joined gpu monitor +2024-05-01 09:09:42,850 INFO HandlerThread:138 [interfaces.py:finish():202] Joined memory monitor +2024-05-01 09:09:42,850 INFO HandlerThread:138 [interfaces.py:finish():202] Joined network monitor +2024-05-01 09:09:42,850 DEBUG SenderThread:138 [sender.py:send():379] send: stats +2024-05-01 09:09:44,429 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: resume +2024-05-01 09:09:44,430 INFO HandlerThread:138 [handler.py:handle_request_resume():699] starting system metrics thread +2024-05-01 09:09:44,430 INFO HandlerThread:138 [system_monitor.py:start():194] Starting system monitor +2024-05-01 09:09:44,430 INFO SystemMonitor:138 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-01 09:09:44,431 INFO SystemMonitor:138 [interfaces.py:start():190] Started cpu monitoring +2024-05-01 09:09:44,432 INFO SystemMonitor:138 [interfaces.py:start():190] Started disk monitoring +2024-05-01 09:09:44,434 INFO SystemMonitor:138 [interfaces.py:start():190] Started gpu monitoring +2024-05-01 09:09:44,435 INFO SystemMonitor:138 [interfaces.py:start():190] Started memory monitoring +2024-05-01 09:09:44,435 INFO SystemMonitor:138 [interfaces.py:start():190] Started network monitoring +2024-05-01 09:09:44,851 DEBUG HandlerThread:138 [handler.py:handle_request():146] handle_request: status_report