diff --git "a/wandb/debug-internal.log" "b/wandb/debug-internal.log" --- "a/wandb/debug-internal.log" +++ "b/wandb/debug-internal.log" @@ -1,32 +1,32 @@ -2024-06-30 20:03:52,246 INFO StreamThr :125 [internal.py:wandb_internal():85] W&B internal server running at pid: 125, started at: 2024-06-30 20:03:52.245629 -2024-06-30 20:03:52,248 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status -2024-06-30 20:03:52,507 INFO WriterThread:125 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/run-ydm2qdsc.wandb -2024-06-30 20:03:52,508 DEBUG SenderThread:125 [sender.py:send():378] send: header -2024-06-30 20:03:52,511 DEBUG SenderThread:125 [sender.py:send():378] send: run -2024-06-30 20:03:52,723 INFO SenderThread:125 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files -2024-06-30 20:03:52,723 INFO SenderThread:125 [sender.py:_start_run_threads():1123] run started: ydm2qdsc with start time 1719777832.246397 -2024-06-30 20:03:52,733 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: check_version -2024-06-30 20:03:52,734 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: check_version -2024-06-30 20:03:52,821 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: run_start -2024-06-30 20:03:52,832 DEBUG HandlerThread:125 [system_info.py:__init__():26] System info init -2024-06-30 20:03:52,832 DEBUG HandlerThread:125 [system_info.py:__init__():41] System info init done -2024-06-30 20:03:52,832 INFO HandlerThread:125 [system_monitor.py:start():194] Starting system monitor -2024-06-30 20:03:52,832 INFO SystemMonitor:125 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-30 20:03:52,832 INFO HandlerThread:125 [system_monitor.py:probe():214] Collecting system info -2024-06-30 20:03:52,833 INFO SystemMonitor:125 [interfaces.py:start():188] Started cpu monitoring -2024-06-30 20:03:52,835 INFO SystemMonitor:125 [interfaces.py:start():188] Started disk monitoring -2024-06-30 20:03:52,835 INFO SystemMonitor:125 [interfaces.py:start():188] Started gpu monitoring -2024-06-30 20:03:52,837 INFO SystemMonitor:125 [interfaces.py:start():188] Started memory monitoring -2024-06-30 20:03:52,837 INFO SystemMonitor:125 [interfaces.py:start():188] Started network monitoring -2024-06-30 20:03:52,845 DEBUG HandlerThread:125 [system_info.py:probe():150] Probing system -2024-06-30 20:03:52,847 DEBUG HandlerThread:125 [gitlib.py:_init_repo():56] git repository is invalid -2024-06-30 20:03:52,847 DEBUG HandlerThread:125 [system_info.py:probe():198] Probing system done -2024-06-30 20:03:52,847 DEBUG HandlerThread:125 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-06-30T20:03:52.845929', 'startedAt': '2024-06-30T20:03:52.239295', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': 'fbbf7acfee4f', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.134, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.134, 'min': 0.0, 'max': 0.0}, {'current': 2000.134, 'min': 0.0, 'max': 0.0}, {'current': 2000.134, 'min': 0.0, 'max': 0.0}, {'current': 2000.134, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5689.845001220703}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357563018798828}} -2024-06-30 20:03:52,847 INFO HandlerThread:125 [system_monitor.py:probe():224] Finished collecting system info -2024-06-30 20:03:52,847 INFO HandlerThread:125 [system_monitor.py:probe():227] Publishing system info -2024-06-30 20:03:52,848 DEBUG HandlerThread:125 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment -2024-06-30 20:03:53,725 INFO Thread-12 :125 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/conda-environment.yaml -2024-06-30 20:04:07,862 ERROR HandlerThread:125 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds +2024-07-03 10:07:00,452 INFO StreamThr :153 [internal.py:wandb_internal():85] W&B internal server running at pid: 153, started at: 2024-07-03 10:07:00.451696 +2024-07-03 10:07:00,453 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status +2024-07-03 10:07:00,954 INFO WriterThread:153 [datastore.py:open_for_write():87] open: /kaggle/working/wandb/run-20240703_100700-cyvdged5/run-cyvdged5.wandb +2024-07-03 10:07:00,955 DEBUG SenderThread:153 [sender.py:send():378] send: header +2024-07-03 10:07:00,958 DEBUG SenderThread:153 [sender.py:send():378] send: run +2024-07-03 10:07:01,527 INFO SenderThread:153 [dir_watcher.py:__init__():211] watching files in: /kaggle/working/wandb/run-20240703_100700-cyvdged5/files +2024-07-03 10:07:01,527 INFO SenderThread:153 [sender.py:_start_run_threads():1123] run started: cyvdged5 with start time 1720001220.452695 +2024-07-03 10:07:01,537 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: check_version +2024-07-03 10:07:01,537 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: check_version +2024-07-03 10:07:01,607 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: run_start +2024-07-03 10:07:01,618 DEBUG HandlerThread:153 [system_info.py:__init__():26] System info init +2024-07-03 10:07:01,618 DEBUG HandlerThread:153 [system_info.py:__init__():41] System info init done +2024-07-03 10:07:01,618 INFO HandlerThread:153 [system_monitor.py:start():194] Starting system monitor +2024-07-03 10:07:01,618 INFO SystemMonitor:153 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-03 10:07:01,618 INFO HandlerThread:153 [system_monitor.py:probe():214] Collecting system info +2024-07-03 10:07:01,619 INFO SystemMonitor:153 [interfaces.py:start():188] Started cpu monitoring +2024-07-03 10:07:01,620 INFO SystemMonitor:153 [interfaces.py:start():188] Started disk monitoring +2024-07-03 10:07:01,621 INFO SystemMonitor:153 [interfaces.py:start():188] Started gpu monitoring +2024-07-03 10:07:01,622 INFO SystemMonitor:153 [interfaces.py:start():188] Started memory monitoring +2024-07-03 10:07:01,623 INFO SystemMonitor:153 [interfaces.py:start():188] Started network monitoring +2024-07-03 10:07:01,631 DEBUG HandlerThread:153 [system_info.py:probe():150] Probing system +2024-07-03 10:07:01,633 DEBUG HandlerThread:153 [gitlib.py:_init_repo():56] git repository is invalid +2024-07-03 10:07:01,633 DEBUG HandlerThread:153 [system_info.py:probe():198] Probing system done +2024-07-03 10:07:01,633 DEBUG HandlerThread:153 [system_monitor.py:probe():223] {'os': 'Linux-5.15.133+-x86_64-with-glibc2.31', 'python': '3.10.13', 'heartbeatAt': '2024-07-03T10:07:01.631454', 'startedAt': '2024-07-03T10:07:00.445303', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': 'kaggle.ipynb', 'codePathLocal': None, 'root': '/kaggle/working', 'host': '2679035f67af', 'username': 'root', 'executable': '/opt/conda/bin/python3.10', 'cpu_count': 2, 'cpu_count_logical': 4, 'cpu_freq': {'current': 2000.21, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 2000.21, 'min': 0.0, 'max': 0.0}, {'current': 2000.21, 'min': 0.0, 'max': 0.0}, {'current': 2000.21, 'min': 0.0, 'max': 0.0}, {'current': 2000.21, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 8062.387607574463, 'used': 5690.048397064209}}, 'gpu': 'Tesla P100-PCIE-16GB', 'gpu_count': 1, 'gpu_devices': [{'name': 'Tesla P100-PCIE-16GB', 'memory_total': 17179869184}], 'memory': {'total': 31.357555389404297}} +2024-07-03 10:07:01,633 INFO HandlerThread:153 [system_monitor.py:probe():224] Finished collecting system info +2024-07-03 10:07:01,633 INFO HandlerThread:153 [system_monitor.py:probe():227] Publishing system info +2024-07-03 10:07:01,633 DEBUG HandlerThread:153 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-07-03 10:07:02,529 INFO Thread-12 :153 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240703_100700-cyvdged5/files/conda-environment.yaml +2024-07-03 10:07:16,647 ERROR HandlerThread:153 [system_info.py:_save_conda():221] Error saving conda packages: Command '['conda', 'env', 'export']' timed out after 15 seconds Traceback (most recent call last): File "/opt/conda/lib/python3.10/site-packages/wandb/sdk/internal/system/system_info.py", line 214, in _save_conda subprocess.call( @@ -37,760 +37,264 @@ Traceback (most recent call last): File "/opt/conda/lib/python3.10/subprocess.py", line 1951, in _wait raise TimeoutExpired(self.args, timeout) subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after 15 seconds -2024-06-30 20:04:07,865 DEBUG HandlerThread:125 [system_info.py:_save_conda():222] Saving conda packages done -2024-06-30 20:04:07,865 INFO HandlerThread:125 [system_monitor.py:probe():229] Finished publishing system info -2024-06-30 20:04:07,873 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:07,873 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: keepalive -2024-06-30 20:04:07,873 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:07,873 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: keepalive -2024-06-30 20:04:07,873 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:07,873 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: keepalive -2024-06-30 20:04:07,874 DEBUG SenderThread:125 [sender.py:send():378] send: files -2024-06-30 20:04:07,874 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-metadata.json with policy now -2024-06-30 20:04:08,272 INFO wandb-upload_0:125 [upload_job.py:push():130] Uploaded file /tmp/tmpq6bhivfrwandb/uw26yi3x-wandb-metadata.json -2024-06-30 20:04:08,728 INFO Thread-12 :125 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-metadata.json -2024-06-30 20:04:08,847 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: python_packages -2024-06-30 20:04:08,847 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: python_packages -2024-06-30 20:04:08,850 DEBUG SenderThread:125 [sender.py:send():378] send: telemetry -2024-06-30 20:04:08,850 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:04:08,851 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:04:08,964 DEBUG SenderThread:125 [sender.py:send():378] send: config -2024-06-30 20:04:08,965 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:04:08,966 DEBUG SenderThread:125 [sender.py:send():378] send: telemetry -2024-06-30 20:04:08,966 DEBUG SenderThread:125 [sender.py:send():378] send: telemetry -2024-06-30 20:04:08,966 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:04:08,966 WARNING SenderThread:125 [sender.py:send_metric():1340] Seen metric with glob (shouldn't happen) -2024-06-30 20:04:08,966 DEBUG SenderThread:125 [sender.py:send():378] send: telemetry -2024-06-30 20:04:09,729 INFO Thread-12 :125 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/requirements.txt -2024-06-30 20:04:12,967 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:17,967 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:22,975 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:23,734 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/config.yaml -2024-06-30 20:04:23,848 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:04:23,849 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:04:28,968 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:33,969 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:38,895 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:04:38,896 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:04:39,053 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:44,054 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:49,055 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:52,838 DEBUG SystemMonitor:125 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2024-06-30 20:04:52,839 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:04:54,131 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:04:54,132 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:04:54,285 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:04:59,285 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:04,286 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:09,287 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:09,350 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:05:09,350 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:05:14,455 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:19,456 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:22,840 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:05:24,598 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:24,598 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:05:24,599 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:05:29,749 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:34,750 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:39,750 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:39,829 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:05:39,829 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:05:44,907 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:49,908 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:52,841 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:05:55,060 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:05:55,061 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:05:55,061 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:06:00,128 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:05,129 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:10,130 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:10,275 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:06:10,276 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:06:15,349 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:20,350 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:22,841 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:06:25,507 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:06:25,507 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:06:25,592 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:30,593 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:35,594 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:40,595 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:40,742 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:06:40,743 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:06:45,903 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:50,904 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:06:52,842 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:06:55,978 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:06:55,979 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:06:56,133 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:01,134 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:06,135 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:11,136 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:11,191 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:07:11,192 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:07:16,306 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:21,306 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:22,843 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:07:26,419 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:07:26,420 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:07:26,573 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:31,574 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:36,574 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:41,575 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:41,655 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:07:41,656 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:07:46,749 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:51,751 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:52,843 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:07:56,844 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:07:56,861 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:07:56,862 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:08:02,028 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:07,029 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:12,030 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:12,083 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:08:12,084 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:08:17,235 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:22,236 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:22,844 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:08:27,082 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:08:27,083 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:08:28,163 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:33,164 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:38,165 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:42,082 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:08:42,083 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:08:42,588 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:08:42,591 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:08:42,591 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:08:42,591 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:08:42,592 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:08:42,592 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:08:42,592 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:08:42,592 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:08:42,592 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:08:42,594 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:08:42,818 INFO Thread-12 :125 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-summary.json -2024-06-30 20:08:42,818 INFO Thread-12 :125 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/output.log -2024-06-30 20:08:43,594 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:44,818 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/output.log -2024-06-30 20:08:48,595 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:52,845 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:08:53,845 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:08:57,303 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:08:57,304 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:08:59,389 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:00,824 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/config.yaml -2024-06-30 20:09:05,120 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:10,120 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:12,544 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:09:12,545 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:09:15,637 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:20,638 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:22,847 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:09:25,848 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:27,776 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:09:27,776 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:09:30,898 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:35,899 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:40,900 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:43,002 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:09:43,003 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:09:46,111 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:51,112 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:52,846 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:09:56,847 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:09:58,229 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:09:58,230 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:10:02,311 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:07,312 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:12,314 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:13,449 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:10:13,450 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:10:17,569 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:22,571 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:22,847 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:10:27,848 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:28,681 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:10:28,682 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:10:33,818 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:38,819 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:43,820 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:43,934 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:10:43,935 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:10:49,067 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:52,848 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:10:54,849 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:10:59,169 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:10:59,169 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:11:00,334 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:05,335 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:10,336 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:14,381 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:11:14,382 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:11:15,530 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:20,531 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:22,849 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:11:25,849 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:29,617 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:11:29,618 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:11:31,754 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:34,393 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:11:34,394 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:11:34,395 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:11:34,395 DEBUG SenderThread:125 [sender.py:send():378] send: metric -2024-06-30 20:11:34,395 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:11:34,395 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:11:34,396 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:11:34,881 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-summary.json -2024-06-30 20:11:37,402 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:37,882 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/config.yaml -2024-06-30 20:11:42,543 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:44,849 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:11:44,849 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:11:47,949 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:52,849 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:11:53,850 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:11:58,851 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:00,093 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:12:00,094 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:12:04,230 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:09,231 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:14,232 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:15,341 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:12:15,342 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:12:19,441 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:22,850 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:12:24,851 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:29,852 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:30,579 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:12:30,580 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:12:35,708 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:40,709 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:45,710 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:45,813 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:12:45,814 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:12:50,967 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:12:52,851 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:12:56,852 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:00,813 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:13:00,814 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:13:01,976 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:06,977 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:11,978 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:15,451 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:13:15,452 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:13:15,452 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:13:15,452 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:13:15,813 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:13:15,814 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:13:15,916 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-summary.json -2024-06-30 20:13:16,916 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/output.log -2024-06-30 20:13:17,975 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:22,852 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:13:23,852 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:28,853 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:30,989 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:13:30,989 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:13:34,067 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:39,068 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:44,068 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:46,217 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:13:46,217 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:13:49,281 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:52,853 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:13:54,854 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:13:59,855 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:01,437 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:14:01,438 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:14:05,593 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:10,594 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:15,594 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:16,702 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:14:16,703 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:14:20,868 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:22,854 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:14:26,855 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:31,855 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:31,935 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:14:31,935 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:14:37,014 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:42,015 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:47,016 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:47,206 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:14:47,207 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:14:52,337 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:14:52,854 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:14:57,856 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:02,438 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:15:02,439 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:15:03,529 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:08,529 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:13,530 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:17,682 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:15:17,682 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:15:18,757 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:22,855 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:15:23,856 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:28,857 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:32,920 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:15:32,920 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:15:33,985 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:38,986 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:43,986 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:48,179 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:15:48,179 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:15:49,327 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:52,856 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:15:54,857 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:15:59,858 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:03,435 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:16:03,436 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:16:05,600 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:10,601 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:15,601 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:18,675 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:16:18,676 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:16:20,751 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:22,857 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:16:25,857 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:30,858 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:33,933 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:16:33,934 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:16:36,085 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:41,086 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:46,087 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:49,177 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:16:49,177 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:16:51,267 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:16:52,857 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:16:56,858 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:01,859 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:04,411 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:17:04,412 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:17:07,531 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:12,532 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:17,533 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:19,413 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:17:19,413 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:17:22,577 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:22,858 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:17:27,859 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:32,860 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:34,419 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:17:34,420 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:17:38,521 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:43,523 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:48,469 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:17:48,470 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:17:48,470 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:17:48,471 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:17:49,012 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-summary.json -2024-06-30 20:17:49,471 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:49,648 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:17:49,648 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:17:50,013 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/output.log -2024-06-30 20:17:52,859 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:17:54,860 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:17:59,860 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:04,861 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:04,897 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:18:04,897 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:18:10,057 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:15,058 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:19,895 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:18:19,896 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:18:20,992 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:22,860 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:18:26,861 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:31,861 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:34,896 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:18:34,896 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:18:37,002 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:42,003 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:47,003 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:49,896 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:18:49,896 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:18:52,860 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:18:52,861 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:18:57,862 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:02,863 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:05,125 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:19:05,125 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:19:08,283 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:13,284 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:18,285 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:20,375 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:19:20,375 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:19:20,919 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:19:20,920 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:19:20,921 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:19:20,921 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:19:21,045 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-summary.json -2024-06-30 20:19:22,861 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:19:23,862 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:28,863 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:33,864 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:35,374 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:19:35,375 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:19:39,515 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:44,516 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:49,517 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:50,374 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:19:50,375 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:19:52,862 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:19:54,863 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:19:59,864 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:04,864 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:05,374 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:20:05,375 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:20:10,477 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:15,478 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:20,479 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:20,524 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:20:20,524 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:20:22,863 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:20:25,864 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:30,864 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:35,763 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:20:35,764 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:20:35,901 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:40,901 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:45,902 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:50,903 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:20:51,020 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:20:51,020 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:20:52,863 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:20:56,864 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:01,865 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:06,275 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:21:06,275 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:21:07,437 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:12,438 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:17,439 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:21,535 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:21:21,535 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:21:22,641 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:22,864 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:21:27,868 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:32,866 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:36,805 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:21:36,805 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:21:37,917 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:42,918 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:47,919 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:52,078 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:21:52,078 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:21:52,865 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:21:53,866 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:21:58,867 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:03,868 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:07,078 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:22:07,078 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:22:09,208 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:14,209 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:19,210 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:21,712 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:22:21,713 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:22:21,713 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:22:21,714 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:22:22,077 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:22:22,078 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:22:22,106 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-summary.json -2024-06-30 20:22:22,866 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:22:24,107 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/output.log -2024-06-30 20:22:24,867 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:29,868 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:34,868 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:37,083 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:22:37,084 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:22:40,236 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:45,237 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:50,238 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:22:52,078 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:22:52,078 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:22:52,867 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:22:55,868 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:00,869 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:05,869 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:07,220 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:23:07,221 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:23:11,324 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:16,325 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:21,325 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:22,220 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:23:22,221 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:23:22,868 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:23:26,869 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:31,870 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:36,871 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:37,220 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:23:37,221 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:23:42,096 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:47,097 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:52,098 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:23:52,406 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:23:52,406 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:23:52,869 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:23:57,870 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:02,871 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:07,404 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:24:07,405 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:24:08,541 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:13,542 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:18,543 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:22,405 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:24:22,405 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:24:22,869 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:24:23,870 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:28,871 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:33,872 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:37,618 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:24:37,619 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:24:39,779 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:44,780 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:49,781 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:52,618 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:24:52,619 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:24:52,870 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:24:54,871 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:24:59,872 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:04,873 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:07,618 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:25:07,618 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:25:10,737 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:15,738 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:20,739 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:22,826 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:25:22,826 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:25:22,988 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:25:25,989 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:30,990 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:35,990 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:37,825 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:25:37,826 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:25:41,895 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:46,895 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:51,896 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:25:52,825 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:25:52,826 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:25:52,986 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:25:56,987 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:01,988 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:06,989 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:08,031 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:26:08,031 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:26:12,207 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:17,208 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:22,209 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:22,872 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:26:23,029 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:26:23,030 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:26:28,122 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:33,123 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:38,030 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:26:38,030 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:26:38,194 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:43,195 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:48,195 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:52,873 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:26:53,030 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:26:53,030 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:26:54,161 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:26:56,483 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:26:56,484 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:26:56,484 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:26:56,485 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:26:57,203 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-summary.json -2024-06-30 20:26:58,203 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/output.log -2024-06-30 20:26:59,485 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:04,486 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:08,030 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:27:08,031 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:27:09,197 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:27:09,198 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:27:09,199 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:27:09,199 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:27:09,207 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-summary.json -2024-06-30 20:27:10,200 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:15,201 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:20,202 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:22,874 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:27:23,080 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:27:23,081 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:27:26,194 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:31,195 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:36,196 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:38,349 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:27:38,350 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:27:41,460 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:46,461 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:51,462 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:27:52,874 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:27:53,606 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:27:53,607 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:27:56,683 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:01,684 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:06,685 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:08,861 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:28:08,862 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:28:11,994 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:16,995 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:21,996 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:22,875 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:28:24,131 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:28:24,132 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:28:27,194 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:32,195 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:37,196 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:39,381 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:28:39,381 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:28:42,531 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:47,532 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:52,533 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:28:52,876 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:28:54,640 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:28:54,640 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:28:57,776 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:02,778 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:07,778 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:09,910 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:29:09,911 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:29:12,999 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:18,000 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:22,877 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:29:23,877 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:25,152 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:29:25,153 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:29:29,270 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:34,271 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:39,272 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:40,419 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:29:40,420 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:29:44,559 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:49,560 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:52,877 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:29:54,878 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:29:55,682 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:29:55,683 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:30:00,837 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:05,838 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:10,839 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:10,928 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:30:10,928 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:30:16,065 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:21,066 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:22,878 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:30:26,189 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:30:26,190 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:30:26,275 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:31,276 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:36,277 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:41,278 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:41,460 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:30:41,461 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:30:46,574 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:51,575 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:52,879 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:30:56,721 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:30:56,722 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:30:56,722 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:31:01,881 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:06,882 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:11,721 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:31:11,721 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:31:12,815 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:17,816 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:22,817 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:22,880 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:31:26,721 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:31:26,722 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:31:27,818 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:29,733 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:31:29,734 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:31:29,734 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:31:29,734 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:31:29,737 DEBUG SenderThread:125 [sender.py:send():378] send: telemetry -2024-06-30 20:31:29,737 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:31:29,739 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: partial_history -2024-06-30 20:31:29,740 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:31:29,741 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:31:29,741 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:31:29,741 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:31:29,742 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: pause -2024-06-30 20:31:29,742 INFO HandlerThread:125 [handler.py:handle_request_pause():724] stopping system metrics thread -2024-06-30 20:31:29,742 INFO HandlerThread:125 [system_monitor.py:finish():203] Stopping system monitor -2024-06-30 20:31:29,742 DEBUG SystemMonitor:125 [system_monitor.py:_start():179] Finished system metrics aggregation loop -2024-06-30 20:31:29,742 DEBUG SystemMonitor:125 [system_monitor.py:_start():183] Publishing last batch of metrics -2024-06-30 20:31:29,744 INFO HandlerThread:125 [interfaces.py:finish():200] Joined cpu monitor -2024-06-30 20:31:29,744 INFO HandlerThread:125 [interfaces.py:finish():200] Joined disk monitor -2024-06-30 20:31:29,744 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:31:29,745 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:31:29,747 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:31:29,748 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:31:29,748 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:31:29,748 DEBUG SenderThread:125 [sender.py:send():378] send: history -2024-06-30 20:31:29,749 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: summary_record -2024-06-30 20:31:29,749 INFO SenderThread:125 [sender.py:_save_file():1389] saving file wandb-summary.json with policy end -2024-06-30 20:31:29,751 INFO HandlerThread:125 [interfaces.py:finish():200] Joined gpu monitor -2024-06-30 20:31:29,752 INFO HandlerThread:125 [interfaces.py:finish():200] Joined memory monitor -2024-06-30 20:31:29,752 INFO HandlerThread:125 [interfaces.py:finish():200] Joined network monitor -2024-06-30 20:31:29,752 DEBUG SenderThread:125 [sender.py:send():378] send: stats -2024-06-30 20:31:30,298 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/wandb-summary.json -2024-06-30 20:31:32,298 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/output.log -2024-06-30 20:31:33,753 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:38,754 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:41,721 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:31:41,721 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:31:43,834 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:48,835 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:53,841 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:31:54,306 INFO Thread-12 :125 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240630_200352-ydm2qdsc/files/config.yaml -2024-06-30 20:31:56,721 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:31:56,722 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:31:59,782 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:04,783 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:09,785 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:11,721 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:32:11,722 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:32:14,833 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:19,835 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:24,836 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:26,722 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:32:26,722 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:32:30,808 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:35,809 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:40,809 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:41,722 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:32:41,723 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:32:45,812 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:50,814 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:55,815 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:32:56,722 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:32:56,723 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:33:01,783 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:33:06,784 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:33:11,723 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:33:11,723 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:33:11,789 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:33:16,790 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:33:21,791 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:33:26,722 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:33:26,723 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:33:26,872 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:33:31,873 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:33:36,477 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: resume -2024-06-30 20:33:36,477 INFO HandlerThread:125 [handler.py:handle_request_resume():715] starting system metrics thread -2024-06-30 20:33:36,477 INFO HandlerThread:125 [system_monitor.py:start():194] Starting system monitor -2024-06-30 20:33:36,477 INFO SystemMonitor:125 [system_monitor.py:_start():158] Starting system asset monitoring threads -2024-06-30 20:33:36,478 INFO SystemMonitor:125 [interfaces.py:start():188] Started cpu monitoring -2024-06-30 20:33:36,478 INFO SystemMonitor:125 [interfaces.py:start():188] Started disk monitoring -2024-06-30 20:33:36,478 INFO SystemMonitor:125 [interfaces.py:start():188] Started gpu monitoring -2024-06-30 20:33:36,479 INFO SystemMonitor:125 [interfaces.py:start():188] Started memory monitoring -2024-06-30 20:33:36,480 INFO SystemMonitor:125 [interfaces.py:start():188] Started network monitoring -2024-06-30 20:33:36,874 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:33:41,723 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: stop_status -2024-06-30 20:33:41,723 DEBUG SenderThread:125 [sender.py:send_request():405] send_request: stop_status -2024-06-30 20:33:42,803 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report -2024-06-30 20:33:47,804 DEBUG HandlerThread:125 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:16,649 DEBUG HandlerThread:153 [system_info.py:_save_conda():222] Saving conda packages done +2024-07-03 10:07:16,650 INFO HandlerThread:153 [system_monitor.py:probe():229] Finished publishing system info +2024-07-03 10:07:16,658 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:16,658 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: keepalive +2024-07-03 10:07:16,658 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:16,658 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: keepalive +2024-07-03 10:07:16,658 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:16,658 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: keepalive +2024-07-03 10:07:16,659 DEBUG SenderThread:153 [sender.py:send():378] send: files +2024-07-03 10:07:16,659 INFO SenderThread:153 [sender.py:_save_file():1389] saving file wandb-metadata.json with policy now +2024-07-03 10:07:17,436 INFO wandb-upload_0:153 [upload_job.py:push():130] Uploaded file /tmp/tmpftsekjz4wandb/0br4hl77-wandb-metadata.json +2024-07-03 10:07:17,532 INFO Thread-12 :153 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240703_100700-cyvdged5/files/wandb-metadata.json +2024-07-03 10:07:17,602 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: python_packages +2024-07-03 10:07:17,603 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: python_packages +2024-07-03 10:07:17,605 DEBUG SenderThread:153 [sender.py:send():378] send: telemetry +2024-07-03 10:07:17,606 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:07:17,607 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:07:17,901 DEBUG SenderThread:153 [sender.py:send():378] send: config +2024-07-03 10:07:17,903 DEBUG SenderThread:153 [sender.py:send():378] send: metric +2024-07-03 10:07:17,904 DEBUG SenderThread:153 [sender.py:send():378] send: telemetry +2024-07-03 10:07:17,904 DEBUG SenderThread:153 [sender.py:send():378] send: telemetry +2024-07-03 10:07:17,904 DEBUG SenderThread:153 [sender.py:send():378] send: metric +2024-07-03 10:07:17,904 WARNING SenderThread:153 [sender.py:send_metric():1340] Seen metric with glob (shouldn't happen) +2024-07-03 10:07:17,904 DEBUG SenderThread:153 [sender.py:send():378] send: telemetry +2024-07-03 10:07:18,532 INFO Thread-12 :153 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240703_100700-cyvdged5/files/requirements.txt +2024-07-03 10:07:20,905 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:25,906 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:30,914 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:31,536 INFO Thread-12 :153 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240703_100700-cyvdged5/files/config.yaml +2024-07-03 10:07:32,603 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:07:32,604 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:07:36,885 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:41,885 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:46,886 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:47,603 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:07:47,604 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:07:52,842 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:07:57,843 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:01,623 DEBUG SystemMonitor:153 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-03 10:08:01,624 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:08:02,604 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:08:02,604 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:08:02,903 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:07,904 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:12,905 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:17,604 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:08:17,604 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:08:18,887 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:23,888 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:28,889 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:31,625 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:08:32,604 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:08:32,605 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:08:34,859 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:39,860 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:44,861 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:47,604 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:08:47,605 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:08:49,909 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:54,910 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:08:59,911 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:01,626 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:09:02,604 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:09:02,605 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:09:04,982 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:09,982 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:14,983 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:17,605 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:09:17,605 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:09:20,857 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:25,858 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:30,859 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:31,627 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:09:32,604 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:09:32,605 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:09:36,837 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:41,838 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:46,839 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:47,657 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:09:47,657 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:09:51,939 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:09:56,940 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:01,627 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:10:02,628 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:02,737 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:10:02,737 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:10:07,966 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:12,967 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:17,737 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:10:17,737 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:10:18,023 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:23,024 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:28,026 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:31,628 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:10:32,946 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:10:32,946 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:10:33,251 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:33,327 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: pause +2024-07-03 10:10:33,328 INFO HandlerThread:153 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-03 10:10:33,328 INFO HandlerThread:153 [system_monitor.py:finish():203] Stopping system monitor +2024-07-03 10:10:33,328 DEBUG SystemMonitor:153 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-03 10:10:33,328 DEBUG SystemMonitor:153 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-03 10:10:33,333 INFO HandlerThread:153 [interfaces.py:finish():200] Joined cpu monitor +2024-07-03 10:10:33,334 INFO HandlerThread:153 [interfaces.py:finish():200] Joined disk monitor +2024-07-03 10:10:33,336 INFO HandlerThread:153 [interfaces.py:finish():200] Joined gpu monitor +2024-07-03 10:10:33,336 INFO HandlerThread:153 [interfaces.py:finish():200] Joined memory monitor +2024-07-03 10:10:33,336 INFO HandlerThread:153 [interfaces.py:finish():200] Joined network monitor +2024-07-03 10:10:33,337 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:10:38,338 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:43,339 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:47,746 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:10:47,747 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:10:49,049 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:54,050 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:10:59,051 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:02,747 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:11:02,747 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:11:05,034 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:10,034 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:15,036 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:17,747 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:11:17,747 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:11:20,051 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:25,052 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:30,053 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:32,747 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:11:32,747 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:11:35,993 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:40,994 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:45,995 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:47,747 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:11:47,747 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:11:51,968 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:11:56,970 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:01,971 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:02,747 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:12:02,748 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:12:07,023 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:12,024 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:17,025 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:17,747 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:12:17,748 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:12:22,062 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:27,064 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:28,052 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: resume +2024-07-03 10:12:28,052 INFO HandlerThread:153 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-03 10:12:28,052 INFO HandlerThread:153 [system_monitor.py:start():194] Starting system monitor +2024-07-03 10:12:28,052 INFO SystemMonitor:153 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-03 10:12:28,053 INFO SystemMonitor:153 [interfaces.py:start():188] Started cpu monitoring +2024-07-03 10:12:28,054 INFO SystemMonitor:153 [interfaces.py:start():188] Started disk monitoring +2024-07-03 10:12:28,056 INFO SystemMonitor:153 [interfaces.py:start():188] Started gpu monitoring +2024-07-03 10:12:28,056 INFO SystemMonitor:153 [interfaces.py:start():188] Started memory monitoring +2024-07-03 10:12:28,057 INFO SystemMonitor:153 [interfaces.py:start():188] Started network monitoring +2024-07-03 10:12:30,640 INFO Thread-12 :153 [dir_watcher.py:_on_file_created():271] file/dir created: /kaggle/working/wandb/run-20240703_100700-cyvdged5/files/output.log +2024-07-03 10:12:31,154 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: pause +2024-07-03 10:12:31,154 INFO HandlerThread:153 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-03 10:12:31,154 INFO HandlerThread:153 [system_monitor.py:finish():203] Stopping system monitor +2024-07-03 10:12:31,154 DEBUG SystemMonitor:153 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-03 10:12:31,154 DEBUG SystemMonitor:153 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-03 10:12:31,155 DEBUG SystemMonitor:153 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-03 10:12:31,156 INFO HandlerThread:153 [interfaces.py:finish():200] Joined cpu monitor +2024-07-03 10:12:31,156 INFO HandlerThread:153 [interfaces.py:finish():200] Joined disk monitor +2024-07-03 10:12:31,161 INFO HandlerThread:153 [interfaces.py:finish():200] Joined gpu monitor +2024-07-03 10:12:31,161 INFO HandlerThread:153 [interfaces.py:finish():200] Joined memory monitor +2024-07-03 10:12:31,161 INFO HandlerThread:153 [interfaces.py:finish():200] Joined network monitor +2024-07-03 10:12:31,162 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:12:32,162 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:32,641 INFO Thread-12 :153 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240703_100700-cyvdged5/files/output.log +2024-07-03 10:12:32,747 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:12:32,748 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:12:36,070 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: resume +2024-07-03 10:12:36,070 INFO HandlerThread:153 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-03 10:12:36,070 INFO HandlerThread:153 [system_monitor.py:start():194] Starting system monitor +2024-07-03 10:12:36,071 INFO SystemMonitor:153 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-03 10:12:36,071 INFO SystemMonitor:153 [interfaces.py:start():188] Started cpu monitoring +2024-07-03 10:12:36,073 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: pause +2024-07-03 10:12:36,074 INFO HandlerThread:153 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-03 10:12:36,074 INFO HandlerThread:153 [system_monitor.py:finish():203] Stopping system monitor +2024-07-03 10:12:36,074 INFO SystemMonitor:153 [interfaces.py:start():188] Started disk monitoring +2024-07-03 10:12:36,074 INFO HandlerThread:153 [interfaces.py:finish():200] Joined cpu monitor +2024-07-03 10:12:36,075 DEBUG SystemMonitor:153 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-03 10:12:36,075 DEBUG SystemMonitor:153 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-03 10:12:36,075 DEBUG SystemMonitor:153 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-03 10:12:36,077 INFO HandlerThread:153 [interfaces.py:finish():200] Joined disk monitor +2024-07-03 10:12:36,078 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:12:38,078 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:39,278 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: resume +2024-07-03 10:12:39,278 INFO HandlerThread:153 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-03 10:12:39,278 INFO HandlerThread:153 [system_monitor.py:start():194] Starting system monitor +2024-07-03 10:12:39,278 INFO SystemMonitor:153 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-03 10:12:39,279 INFO SystemMonitor:153 [interfaces.py:start():188] Started cpu monitoring +2024-07-03 10:12:39,280 INFO SystemMonitor:153 [interfaces.py:start():188] Started disk monitoring +2024-07-03 10:12:39,282 INFO SystemMonitor:153 [interfaces.py:start():188] Started gpu monitoring +2024-07-03 10:12:39,282 INFO SystemMonitor:153 [interfaces.py:start():188] Started memory monitoring +2024-07-03 10:12:39,284 INFO SystemMonitor:153 [interfaces.py:start():188] Started network monitoring +2024-07-03 10:12:40,032 DEBUG SenderThread:153 [sender.py:send():378] send: config +2024-07-03 10:12:40,034 DEBUG SenderThread:153 [sender.py:send():378] send: metric +2024-07-03 10:12:40,034 DEBUG SenderThread:153 [sender.py:send():378] send: metric +2024-07-03 10:12:40,034 WARNING SenderThread:153 [sender.py:send_metric():1340] Seen metric with glob (shouldn't happen) +2024-07-03 10:12:44,040 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:44,645 INFO Thread-12 :153 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240703_100700-cyvdged5/files/config.yaml +2024-07-03 10:12:47,748 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:12:47,748 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:12:50,014 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:12:55,015 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:00,016 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:02,748 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:13:02,748 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:13:06,006 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:11,007 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:16,008 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:17,748 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:13:17,748 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:13:22,000 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:27,001 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:32,002 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:32,748 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:13:32,748 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:13:37,018 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:39,284 DEBUG SystemMonitor:153 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-07-03 10:13:39,285 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:13:42,286 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:47,288 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:47,748 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:13:47,748 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:13:52,987 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:55,033 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: pause +2024-07-03 10:13:55,033 INFO HandlerThread:153 [handler.py:handle_request_pause():724] stopping system metrics thread +2024-07-03 10:13:55,033 INFO HandlerThread:153 [system_monitor.py:finish():203] Stopping system monitor +2024-07-03 10:13:55,033 DEBUG SystemMonitor:153 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-07-03 10:13:55,033 DEBUG SystemMonitor:153 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-07-03 10:13:55,034 INFO HandlerThread:153 [interfaces.py:finish():200] Joined cpu monitor +2024-07-03 10:13:55,035 INFO HandlerThread:153 [interfaces.py:finish():200] Joined disk monitor +2024-07-03 10:13:55,041 INFO HandlerThread:153 [interfaces.py:finish():200] Joined gpu monitor +2024-07-03 10:13:55,041 INFO HandlerThread:153 [interfaces.py:finish():200] Joined memory monitor +2024-07-03 10:13:55,041 INFO HandlerThread:153 [interfaces.py:finish():200] Joined network monitor +2024-07-03 10:13:55,042 DEBUG SenderThread:153 [sender.py:send():378] send: stats +2024-07-03 10:13:58,043 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:13:58,120 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: resume +2024-07-03 10:13:58,120 INFO HandlerThread:153 [handler.py:handle_request_resume():715] starting system metrics thread +2024-07-03 10:13:58,120 INFO HandlerThread:153 [system_monitor.py:start():194] Starting system monitor +2024-07-03 10:13:58,120 INFO SystemMonitor:153 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-07-03 10:13:58,121 INFO SystemMonitor:153 [interfaces.py:start():188] Started cpu monitoring +2024-07-03 10:13:58,121 INFO SystemMonitor:153 [interfaces.py:start():188] Started disk monitoring +2024-07-03 10:13:58,122 INFO SystemMonitor:153 [interfaces.py:start():188] Started gpu monitoring +2024-07-03 10:13:58,125 INFO SystemMonitor:153 [interfaces.py:start():188] Started memory monitoring +2024-07-03 10:13:58,126 INFO SystemMonitor:153 [interfaces.py:start():188] Started network monitoring +2024-07-03 10:14:02,749 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:14:02,749 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:14:03,238 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:14:08,239 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:14:13,240 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:14:17,748 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:14:17,749 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:14:19,050 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:14:24,051 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:14:29,053 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:14:32,748 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: stop_status +2024-07-03 10:14:32,749 DEBUG SenderThread:153 [sender.py:send_request():405] send_request: stop_status +2024-07-03 10:14:34,055 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report +2024-07-03 10:14:39,056 DEBUG HandlerThread:153 [handler.py:handle_request():158] handle_request: status_report