Training in progress, epoch 3

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:444aacd5638163bab056fa7df4012bbce8bd6b481ed9b15c2e41fa88abaf95a1
+oid sha256:08ea55e44418f1f804def2fc87125f2fae57802b41ef66ecb747fef7deda96f7
 size 502675828

runs/Mar07_03-35-46_8bb175de6f67/events.out.tfevents.1709782548.8bb175de6f67.34.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cb1d304e3a72ed7a6c1f2f056a1a2ea7ba752455afb80e62b643040e693af26d
-size 6507
+oid sha256:2ad12ac342b33e561449e00d6874ef3f2c79386c6ae8c2ca9d8745f123346f83
+size 7190

wandb/debug-internal.log

@@ -127,3 +127,30 @@
 2024-03-07 03:37:46,913 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
 2024-03-07 03:37:50,880 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-07 03:37:51,914 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:37:55,881 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:37:56,915 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:00,882 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:01,916 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:05,517 DEBUG   SenderThread:147 [sender.py:send():382] send: stats
+2024-03-07 03:38:05,887 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:07,518 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:10,888 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:11,123 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-07 03:38:11,124 DEBUG   SenderThread:147 [sender.py:send():382] send: history
+2024-03-07 03:38:11,124 DEBUG   SenderThread:147 [sender.py:send_request():409] send_request: summary_record
+2024-03-07 03:38:11,125 INFO    SenderThread:147 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-07 03:38:11,486 INFO    Thread-12 :147 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240307_033604-u9myy8y3/files/wandb-summary.json
+2024-03-07 03:38:12,241 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-07 03:38:12,242 DEBUG   SenderThread:147 [sender.py:send():382] send: history
+2024-03-07 03:38:12,242 DEBUG   SenderThread:147 [sender.py:send_request():409] send_request: summary_record
+2024-03-07 03:38:12,243 INFO    SenderThread:147 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-07 03:38:12,486 INFO    Thread-12 :147 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240307_033604-u9myy8y3/files/wandb-summary.json
+2024-03-07 03:38:13,243 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:14,487 INFO    Thread-12 :147 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240307_033604-u9myy8y3/files/output.log
+2024-03-07 03:38:16,244 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:18,244 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:21,245 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:23,245 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:26,246 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:28,246 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:31,247 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive

wandb/run-20240307_033604-u9myy8y3/files/output.log

@@ -1,3 +1,5 @@
+/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
   _warn_prf(average, modifier, msg_start, len(result))
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.

wandb/run-20240307_033604-u9myy8y3/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.6989, "train/grad_norm": 9.77726936340332, "train/learning_rate": 1.2064516129032259e-05, "train/epoch": 2.0, "train/global_step": 124, "_timestamp": 1709782657.9073257, "_runtime": 93.89001870155334, "_step": 3, "eval/loss": 1.758449912071228, "eval/accuracy": 0.36363636363636365, "eval/precision": 0.19529933481152995, "eval/recall": 0.36363636363636365, "eval/f1": 0.24694214876033058, "eval/runtime": 1.1141, "eval/samples_per_second": 49.368, "eval/steps_per_second": 6.283}
+{"train/loss": 1.6246, "train/grad_norm": 9.643479347229004, "train/learning_rate": 8.064516129032258e-06, "train/epoch": 3.0, "train/global_step": 186, "_timestamp": 1709782692.2407324, "_runtime": 128.22342538833618, "_step": 5, "eval/loss": 1.7605557441711426, "eval/accuracy": 0.41818181818181815, "eval/precision": 0.2806526806526807, "eval/recall": 0.41818181818181815, "eval/f1": 0.31359361641493305, "eval/runtime": 1.1141, "eval/samples_per_second": 49.369, "eval/steps_per_second": 6.283}

wandb/run-20240307_033604-u9myy8y3/logs/debug-internal.log

@@ -127,3 +127,30 @@
 2024-03-07 03:37:46,913 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
 2024-03-07 03:37:50,880 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-07 03:37:51,914 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:37:55,881 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:37:56,915 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:00,882 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:01,916 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:05,517 DEBUG   SenderThread:147 [sender.py:send():382] send: stats
+2024-03-07 03:38:05,887 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:07,518 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:10,888 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:11,123 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-07 03:38:11,124 DEBUG   SenderThread:147 [sender.py:send():382] send: history
+2024-03-07 03:38:11,124 DEBUG   SenderThread:147 [sender.py:send_request():409] send_request: summary_record
+2024-03-07 03:38:11,125 INFO    SenderThread:147 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-07 03:38:11,486 INFO    Thread-12 :147 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240307_033604-u9myy8y3/files/wandb-summary.json
+2024-03-07 03:38:12,241 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-07 03:38:12,242 DEBUG   SenderThread:147 [sender.py:send():382] send: history
+2024-03-07 03:38:12,242 DEBUG   SenderThread:147 [sender.py:send_request():409] send_request: summary_record
+2024-03-07 03:38:12,243 INFO    SenderThread:147 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-07 03:38:12,486 INFO    Thread-12 :147 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240307_033604-u9myy8y3/files/wandb-summary.json
+2024-03-07 03:38:13,243 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:14,487 INFO    Thread-12 :147 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240307_033604-u9myy8y3/files/output.log
+2024-03-07 03:38:16,244 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:18,244 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:21,245 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:23,245 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:26,246 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-07 03:38:28,246 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: status_report
+2024-03-07 03:38:31,247 DEBUG   HandlerThread:147 [handler.py:handle_request():146] handle_request: keepalive