Training in progress, epoch 2

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6b0691cfcc8367001de4c6ef5342c5f4d7acedc2d07f4d7efbdc8912bd683cf
+oid sha256:cfe6d338329da55090336cbedc4b5a2e2cbf1a2af923c0c31cc60736ee6be81a
 size 502675828

runs/Apr10_14-58-13_9088970cc926/events.out.tfevents.1712761095.9088970cc926.34.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d28013de0a8616b50715aff1fb4caae6b65ff5a0f0ff7f0f06ad3e9a240e4db1
-size 5843
+oid sha256:fc627c60d0dbda5a7fe0a71077d686b93729261151758ad3bc048c71add6f7e7
+size 6526

wandb/debug-internal.log

@@ -133,3 +133,28 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-10 15:03:26,753 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
 2024-04-10 15:03:26,754 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
 2024-04-10 15:03:26,794 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:03:29,980 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:34,981 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:36,833 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:03:36,835 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:03:36,835 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:03:36,837 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:03:37,698 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:03:39,680 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:03:39,681 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:03:39,682 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:03:39,682 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:03:39,699 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:03:40,683 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:41,699 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log
+2024-04-10 15:03:42,006 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:03:42,039 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:03:42,039 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:03:46,231 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:51,232 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:55,777 DEBUG   SenderThread:100 [sender.py:send():379] send: stats
+2024-04-10 15:03:56,778 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:56,824 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:03:56,824 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:03:56,832 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:04:02,043 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240410_150123-cuua991r/files/output.log

@@ -1 +1,3 @@
+/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.

wandb/run-20240410_150123-cuua991r/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.9109, "train/grad_norm": 8.288476943969727, "train/learning_rate": 1.6033898305084747e-05, "train/epoch": 1.0, "train/global_step": 118, "_timestamp": 1712761359.6090517, "_runtime": 76.05831980705261, "_step": 1, "eval/loss": 1.8787012100219727, "eval/accuracy": 0.24647887323943662, "eval/precision": 0.1420832688438322, "eval/recall": 0.24647887323943662, "eval/f1": 0.17390879409993346, "eval/runtime": 2.8503, "eval/samples_per_second": 49.82, "eval/steps_per_second": 6.315}
+{"train/loss": 1.8337, "train/grad_norm": 8.119162559509277, "train/learning_rate": 1.2033898305084745e-05, "train/epoch": 2.0, "train/global_step": 236, "_timestamp": 1712761419.6800258, "_runtime": 136.12929391860962, "_step": 3, "eval/loss": 1.8552484512329102, "eval/accuracy": 0.2605633802816901, "eval/precision": 0.229201952219647, "eval/recall": 0.2605633802816901, "eval/f1": 0.20020510365383345, "eval/runtime": 2.8426, "eval/samples_per_second": 49.954, "eval/steps_per_second": 6.332}

wandb/run-20240410_150123-cuua991r/logs/debug-internal.log

@@ -133,3 +133,28 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-10 15:03:26,753 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
 2024-04-10 15:03:26,754 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
 2024-04-10 15:03:26,794 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:03:29,980 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:34,981 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:36,833 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:03:36,835 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:03:36,835 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:03:36,837 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:03:37,698 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:03:39,680 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:03:39,681 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:03:39,682 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:03:39,682 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:03:39,699 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:03:40,683 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:41,699 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log
+2024-04-10 15:03:42,006 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:03:42,039 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:03:42,039 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:03:46,231 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:51,232 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:55,777 DEBUG   SenderThread:100 [sender.py:send():379] send: stats
+2024-04-10 15:03:56,778 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:03:56,824 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:03:56,824 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:03:56,832 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:04:02,043 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report