reward-train-facebook-opt350m_v2
Browse files
wandb/debug-internal.log
CHANGED
|
@@ -6629,3 +6629,47 @@
|
|
| 6629 |
2024-06-12 19:19:03,787 INFO SystemMonitor:143 [interfaces.py:start():190] Started gpu monitoring
|
| 6630 |
2024-06-12 19:19:03,788 INFO SystemMonitor:143 [interfaces.py:start():190] Started memory monitoring
|
| 6631 |
2024-06-12 19:19:03,788 INFO SystemMonitor:143 [interfaces.py:start():190] Started network monitoring
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6629 |
2024-06-12 19:19:03,787 INFO SystemMonitor:143 [interfaces.py:start():190] Started gpu monitoring
|
| 6630 |
2024-06-12 19:19:03,788 INFO SystemMonitor:143 [interfaces.py:start():190] Started memory monitoring
|
| 6631 |
2024-06-12 19:19:03,788 INFO SystemMonitor:143 [interfaces.py:start():190] Started network monitoring
|
| 6632 |
+
2024-06-12 19:19:04,786 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: status_report
|
| 6633 |
+
2024-06-12 19:19:06,044 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: keepalive
|
| 6634 |
+
2024-06-12 19:19:07,056 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: pause
|
| 6635 |
+
2024-06-12 19:19:07,056 INFO HandlerThread:143 [handler.py:handle_request_pause():708] stopping system metrics thread
|
| 6636 |
+
2024-06-12 19:19:07,056 INFO HandlerThread:143 [system_monitor.py:finish():203] Stopping system monitor
|
| 6637 |
+
2024-06-12 19:19:07,057 INFO HandlerThread:143 [interfaces.py:finish():202] Joined cpu monitor
|
| 6638 |
+
2024-06-12 19:19:07,057 DEBUG SystemMonitor:143 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 6639 |
+
2024-06-12 19:19:07,057 DEBUG SystemMonitor:143 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 6640 |
+
2024-06-12 19:19:07,057 DEBUG SystemMonitor:143 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 6641 |
+
2024-06-12 19:19:07,058 INFO HandlerThread:143 [interfaces.py:finish():202] Joined disk monitor
|
| 6642 |
+
2024-06-12 19:19:07,064 INFO HandlerThread:143 [interfaces.py:finish():202] Joined gpu monitor
|
| 6643 |
+
2024-06-12 19:19:07,064 INFO HandlerThread:143 [interfaces.py:finish():202] Joined memory monitor
|
| 6644 |
+
2024-06-12 19:19:07,064 INFO HandlerThread:143 [interfaces.py:finish():202] Joined network monitor
|
| 6645 |
+
2024-06-12 19:19:07,065 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: resume
|
| 6646 |
+
2024-06-12 19:19:07,066 DEBUG SenderThread:143 [sender.py:send():382] send: stats
|
| 6647 |
+
2024-06-12 19:19:07,066 INFO HandlerThread:143 [handler.py:handle_request_resume():699] starting system metrics thread
|
| 6648 |
+
2024-06-12 19:19:07,066 INFO HandlerThread:143 [system_monitor.py:start():194] Starting system monitor
|
| 6649 |
+
2024-06-12 19:19:07,066 INFO SystemMonitor:143 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 6650 |
+
2024-06-12 19:19:07,067 INFO SystemMonitor:143 [interfaces.py:start():190] Started cpu monitoring
|
| 6651 |
+
2024-06-12 19:19:07,068 INFO SystemMonitor:143 [interfaces.py:start():190] Started disk monitoring
|
| 6652 |
+
2024-06-12 19:19:07,068 INFO SystemMonitor:143 [interfaces.py:start():190] Started gpu monitoring
|
| 6653 |
+
2024-06-12 19:19:07,069 INFO SystemMonitor:143 [interfaces.py:start():190] Started memory monitoring
|
| 6654 |
+
2024-06-12 19:19:07,070 INFO SystemMonitor:143 [interfaces.py:start():190] Started network monitoring
|
| 6655 |
+
2024-06-12 19:19:08,097 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: pause
|
| 6656 |
+
2024-06-12 19:19:08,097 INFO HandlerThread:143 [handler.py:handle_request_pause():708] stopping system metrics thread
|
| 6657 |
+
2024-06-12 19:19:08,097 INFO HandlerThread:143 [system_monitor.py:finish():203] Stopping system monitor
|
| 6658 |
+
2024-06-12 19:19:08,097 DEBUG SystemMonitor:143 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 6659 |
+
2024-06-12 19:19:08,098 DEBUG SystemMonitor:143 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 6660 |
+
2024-06-12 19:19:08,098 DEBUG SystemMonitor:143 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 6661 |
+
2024-06-12 19:19:08,098 INFO HandlerThread:143 [interfaces.py:finish():202] Joined cpu monitor
|
| 6662 |
+
2024-06-12 19:19:08,099 INFO HandlerThread:143 [interfaces.py:finish():202] Joined disk monitor
|
| 6663 |
+
2024-06-12 19:19:08,105 INFO HandlerThread:143 [interfaces.py:finish():202] Joined gpu monitor
|
| 6664 |
+
2024-06-12 19:19:08,106 INFO HandlerThread:143 [interfaces.py:finish():202] Joined memory monitor
|
| 6665 |
+
2024-06-12 19:19:08,106 INFO HandlerThread:143 [interfaces.py:finish():202] Joined network monitor
|
| 6666 |
+
2024-06-12 19:19:08,106 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: resume
|
| 6667 |
+
2024-06-12 19:19:08,106 INFO HandlerThread:143 [handler.py:handle_request_resume():699] starting system metrics thread
|
| 6668 |
+
2024-06-12 19:19:08,106 INFO HandlerThread:143 [system_monitor.py:start():194] Starting system monitor
|
| 6669 |
+
2024-06-12 19:19:08,106 INFO SystemMonitor:143 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 6670 |
+
2024-06-12 19:19:08,107 INFO SystemMonitor:143 [interfaces.py:start():190] Started cpu monitoring
|
| 6671 |
+
2024-06-12 19:19:08,107 DEBUG SenderThread:143 [sender.py:send():382] send: stats
|
| 6672 |
+
2024-06-12 19:19:08,108 INFO SystemMonitor:143 [interfaces.py:start():190] Started disk monitoring
|
| 6673 |
+
2024-06-12 19:19:08,109 INFO SystemMonitor:143 [interfaces.py:start():190] Started gpu monitoring
|
| 6674 |
+
2024-06-12 19:19:08,109 INFO SystemMonitor:143 [interfaces.py:start():190] Started memory monitoring
|
| 6675 |
+
2024-06-12 19:19:08,109 INFO SystemMonitor:143 [interfaces.py:start():190] Started network monitoring
|
wandb/debug.log
CHANGED
|
@@ -38,3 +38,9 @@ config: {}
|
|
| 38 |
2024-06-12 19:19:03,775 INFO MainThread:26 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 39 |
2024-06-12 19:19:03,776 INFO MainThread:26 [wandb_init.py:_pause_backend():437] pausing backend
|
| 40 |
2024-06-12 19:19:03,781 INFO MainThread:26 [wandb_init.py:_resume_backend():442] resuming backend
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
2024-06-12 19:19:03,775 INFO MainThread:26 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 39 |
2024-06-12 19:19:03,776 INFO MainThread:26 [wandb_init.py:_pause_backend():437] pausing backend
|
| 40 |
2024-06-12 19:19:03,781 INFO MainThread:26 [wandb_init.py:_resume_backend():442] resuming backend
|
| 41 |
+
2024-06-12 19:19:07,055 INFO MainThread:26 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 42 |
+
2024-06-12 19:19:07,056 INFO MainThread:26 [wandb_init.py:_pause_backend():437] pausing backend
|
| 43 |
+
2024-06-12 19:19:07,063 INFO MainThread:26 [wandb_init.py:_resume_backend():442] resuming backend
|
| 44 |
+
2024-06-12 19:19:08,096 INFO MainThread:26 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 45 |
+
2024-06-12 19:19:08,096 INFO MainThread:26 [wandb_init.py:_pause_backend():437] pausing backend
|
| 46 |
+
2024-06-12 19:19:08,101 INFO MainThread:26 [wandb_init.py:_resume_backend():442] resuming backend
|
wandb/run-20240612_150610-7vg9fga2/logs/debug-internal.log
CHANGED
|
@@ -6629,3 +6629,47 @@
|
|
| 6629 |
2024-06-12 19:19:03,787 INFO SystemMonitor:143 [interfaces.py:start():190] Started gpu monitoring
|
| 6630 |
2024-06-12 19:19:03,788 INFO SystemMonitor:143 [interfaces.py:start():190] Started memory monitoring
|
| 6631 |
2024-06-12 19:19:03,788 INFO SystemMonitor:143 [interfaces.py:start():190] Started network monitoring
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 6629 |
2024-06-12 19:19:03,787 INFO SystemMonitor:143 [interfaces.py:start():190] Started gpu monitoring
|
| 6630 |
2024-06-12 19:19:03,788 INFO SystemMonitor:143 [interfaces.py:start():190] Started memory monitoring
|
| 6631 |
2024-06-12 19:19:03,788 INFO SystemMonitor:143 [interfaces.py:start():190] Started network monitoring
|
| 6632 |
+
2024-06-12 19:19:04,786 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: status_report
|
| 6633 |
+
2024-06-12 19:19:06,044 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: keepalive
|
| 6634 |
+
2024-06-12 19:19:07,056 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: pause
|
| 6635 |
+
2024-06-12 19:19:07,056 INFO HandlerThread:143 [handler.py:handle_request_pause():708] stopping system metrics thread
|
| 6636 |
+
2024-06-12 19:19:07,056 INFO HandlerThread:143 [system_monitor.py:finish():203] Stopping system monitor
|
| 6637 |
+
2024-06-12 19:19:07,057 INFO HandlerThread:143 [interfaces.py:finish():202] Joined cpu monitor
|
| 6638 |
+
2024-06-12 19:19:07,057 DEBUG SystemMonitor:143 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 6639 |
+
2024-06-12 19:19:07,057 DEBUG SystemMonitor:143 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 6640 |
+
2024-06-12 19:19:07,057 DEBUG SystemMonitor:143 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 6641 |
+
2024-06-12 19:19:07,058 INFO HandlerThread:143 [interfaces.py:finish():202] Joined disk monitor
|
| 6642 |
+
2024-06-12 19:19:07,064 INFO HandlerThread:143 [interfaces.py:finish():202] Joined gpu monitor
|
| 6643 |
+
2024-06-12 19:19:07,064 INFO HandlerThread:143 [interfaces.py:finish():202] Joined memory monitor
|
| 6644 |
+
2024-06-12 19:19:07,064 INFO HandlerThread:143 [interfaces.py:finish():202] Joined network monitor
|
| 6645 |
+
2024-06-12 19:19:07,065 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: resume
|
| 6646 |
+
2024-06-12 19:19:07,066 DEBUG SenderThread:143 [sender.py:send():382] send: stats
|
| 6647 |
+
2024-06-12 19:19:07,066 INFO HandlerThread:143 [handler.py:handle_request_resume():699] starting system metrics thread
|
| 6648 |
+
2024-06-12 19:19:07,066 INFO HandlerThread:143 [system_monitor.py:start():194] Starting system monitor
|
| 6649 |
+
2024-06-12 19:19:07,066 INFO SystemMonitor:143 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 6650 |
+
2024-06-12 19:19:07,067 INFO SystemMonitor:143 [interfaces.py:start():190] Started cpu monitoring
|
| 6651 |
+
2024-06-12 19:19:07,068 INFO SystemMonitor:143 [interfaces.py:start():190] Started disk monitoring
|
| 6652 |
+
2024-06-12 19:19:07,068 INFO SystemMonitor:143 [interfaces.py:start():190] Started gpu monitoring
|
| 6653 |
+
2024-06-12 19:19:07,069 INFO SystemMonitor:143 [interfaces.py:start():190] Started memory monitoring
|
| 6654 |
+
2024-06-12 19:19:07,070 INFO SystemMonitor:143 [interfaces.py:start():190] Started network monitoring
|
| 6655 |
+
2024-06-12 19:19:08,097 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: pause
|
| 6656 |
+
2024-06-12 19:19:08,097 INFO HandlerThread:143 [handler.py:handle_request_pause():708] stopping system metrics thread
|
| 6657 |
+
2024-06-12 19:19:08,097 INFO HandlerThread:143 [system_monitor.py:finish():203] Stopping system monitor
|
| 6658 |
+
2024-06-12 19:19:08,097 DEBUG SystemMonitor:143 [system_monitor.py:_start():172] Starting system metrics aggregation loop
|
| 6659 |
+
2024-06-12 19:19:08,098 DEBUG SystemMonitor:143 [system_monitor.py:_start():179] Finished system metrics aggregation loop
|
| 6660 |
+
2024-06-12 19:19:08,098 DEBUG SystemMonitor:143 [system_monitor.py:_start():183] Publishing last batch of metrics
|
| 6661 |
+
2024-06-12 19:19:08,098 INFO HandlerThread:143 [interfaces.py:finish():202] Joined cpu monitor
|
| 6662 |
+
2024-06-12 19:19:08,099 INFO HandlerThread:143 [interfaces.py:finish():202] Joined disk monitor
|
| 6663 |
+
2024-06-12 19:19:08,105 INFO HandlerThread:143 [interfaces.py:finish():202] Joined gpu monitor
|
| 6664 |
+
2024-06-12 19:19:08,106 INFO HandlerThread:143 [interfaces.py:finish():202] Joined memory monitor
|
| 6665 |
+
2024-06-12 19:19:08,106 INFO HandlerThread:143 [interfaces.py:finish():202] Joined network monitor
|
| 6666 |
+
2024-06-12 19:19:08,106 DEBUG HandlerThread:143 [handler.py:handle_request():146] handle_request: resume
|
| 6667 |
+
2024-06-12 19:19:08,106 INFO HandlerThread:143 [handler.py:handle_request_resume():699] starting system metrics thread
|
| 6668 |
+
2024-06-12 19:19:08,106 INFO HandlerThread:143 [system_monitor.py:start():194] Starting system monitor
|
| 6669 |
+
2024-06-12 19:19:08,106 INFO SystemMonitor:143 [system_monitor.py:_start():158] Starting system asset monitoring threads
|
| 6670 |
+
2024-06-12 19:19:08,107 INFO SystemMonitor:143 [interfaces.py:start():190] Started cpu monitoring
|
| 6671 |
+
2024-06-12 19:19:08,107 DEBUG SenderThread:143 [sender.py:send():382] send: stats
|
| 6672 |
+
2024-06-12 19:19:08,108 INFO SystemMonitor:143 [interfaces.py:start():190] Started disk monitoring
|
| 6673 |
+
2024-06-12 19:19:08,109 INFO SystemMonitor:143 [interfaces.py:start():190] Started gpu monitoring
|
| 6674 |
+
2024-06-12 19:19:08,109 INFO SystemMonitor:143 [interfaces.py:start():190] Started memory monitoring
|
| 6675 |
+
2024-06-12 19:19:08,109 INFO SystemMonitor:143 [interfaces.py:start():190] Started network monitoring
|
wandb/run-20240612_150610-7vg9fga2/logs/debug.log
CHANGED
|
@@ -38,3 +38,9 @@ config: {}
|
|
| 38 |
2024-06-12 19:19:03,775 INFO MainThread:26 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 39 |
2024-06-12 19:19:03,776 INFO MainThread:26 [wandb_init.py:_pause_backend():437] pausing backend
|
| 40 |
2024-06-12 19:19:03,781 INFO MainThread:26 [wandb_init.py:_resume_backend():442] resuming backend
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
2024-06-12 19:19:03,775 INFO MainThread:26 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 39 |
2024-06-12 19:19:03,776 INFO MainThread:26 [wandb_init.py:_pause_backend():437] pausing backend
|
| 40 |
2024-06-12 19:19:03,781 INFO MainThread:26 [wandb_init.py:_resume_backend():442] resuming backend
|
| 41 |
+
2024-06-12 19:19:07,055 INFO MainThread:26 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 42 |
+
2024-06-12 19:19:07,056 INFO MainThread:26 [wandb_init.py:_pause_backend():437] pausing backend
|
| 43 |
+
2024-06-12 19:19:07,063 INFO MainThread:26 [wandb_init.py:_resume_backend():442] resuming backend
|
| 44 |
+
2024-06-12 19:19:08,096 INFO MainThread:26 [jupyter.py:save_ipynb():373] not saving jupyter notebook
|
| 45 |
+
2024-06-12 19:19:08,096 INFO MainThread:26 [wandb_init.py:_pause_backend():437] pausing backend
|
| 46 |
+
2024-06-12 19:19:08,101 INFO MainThread:26 [wandb_init.py:_resume_backend():442] resuming backend
|