Training in progress, epoch 6

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f92cf0b63337107efe8f72177c57bb836f176cfa48fc5d52a3213683c7fcb2e3
+oid sha256:48d54688d8b764d3c5333c7272a5be314c35808115c5871f0b9b81372680d5d9
 size 502675828

runs/Mar06_15-06-49_41759fa8e6ad/events.out.tfevents.1709737609.41759fa8e6ad.34.4

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d48f1729e2ef03bdaceb8ff419b66a5cc257fe32d744152ebd794b459fb6766
-size 8533
+oid sha256:99f23f9353b63b66080a7b7ea1a95886a34da85a4609e73ac049ebcfe18b2ea1
+size 9216

wandb/debug-internal.log

@@ -833,3 +833,36 @@
 2024-03-06 15:09:43,756 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:09:43,797 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:09:45,364 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:48,757 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:48,798 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:50,365 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:53,758 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:53,799 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:55,366 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:56,802 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:09:58,769 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:58,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:00,367 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:02,535 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:10:02,537 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:10:02,537 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:10:02,537 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:10:03,108 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:10:03,653 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:10:03,655 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:10:03,655 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:10:03,656 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:10:03,826 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:04,108 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:10:04,657 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:05,368 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:06,109 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:10:08,851 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:09,658 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:10,368 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:13,851 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:14,659 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:15,369 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:18,852 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:19,660 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:20,370 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240306_145424-trm7fvg4/logs/debug-internal.log

@@ -884,3 +884,36 @@ wandb.errors.AuthenticationError: The API key you provided is either invalid or
 2024-03-06 15:09:43,756 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:09:43,797 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:09:45,364 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:48,757 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:48,798 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:50,365 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:53,758 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:53,799 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:55,366 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:56,802 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:09:58,769 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:58,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:00,367 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:02,535 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:10:02,537 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:10:02,537 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:10:02,537 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:10:03,108 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:10:03,653 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:10:03,655 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:10:03,655 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:10:03,656 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:10:03,826 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:04,108 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:10:04,657 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:05,368 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:06,109 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:10:08,851 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:09,658 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:10,368 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:13,851 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:14,659 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:15,369 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:18,852 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:19,660 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:20,370 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report

wandb/run-20240306_145455-h1uv5tyi/files/output.log

@@ -46,3 +46,6 @@ Checkpoint destination directory /kaggle/working/checkpoint-62 already exists an
   _warn_prf(average, modifier, msg_start, len(result))
 Checkpoint destination directory /kaggle/working/checkpoint-124 already exists and is non-empty. Saving will proceed but saved results may be invalid.
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))
+/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))

wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.6049, "train/grad_norm": NaN, "train/learning_rate": 1.5096774193548387e-05, "train/epoch": 5.0, "train/global_step": 155, "_timestamp": 1709737769.791673, "_runtime": 874.0729160308838, "_step": 35, "eval/loss": 1.730015516281128, "eval/accuracy": 0.36363636363636365, "eval/precision": 0.19892561983471074, "eval/recall": 0.36363636363636365, "eval/f1": 0.2459399186671914, "eval/runtime": 1.1125, "eval/samples_per_second": 49.439, "eval/steps_per_second": 3.596, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}
+{"train/loss": 1.5571, "train/grad_norm": 8.44052505493164, "train/learning_rate": 1.2096774193548387e-05, "train/epoch": 6.0, "train/global_step": 186, "_timestamp": 1709737803.6531816, "_runtime": 907.9344246387482, "_step": 37, "eval/loss": 1.7199132442474365, "eval/accuracy": 0.41818181818181815, "eval/precision": 0.38225108225108223, "eval/recall": 0.41818181818181815, "eval/f1": 0.3338471364700873, "eval/runtime": 1.1139, "eval/samples_per_second": 49.377, "eval/steps_per_second": 3.591, "train/train_runtime": 237.4526, "train/train_samples_per_second": 10.339, "train/train_steps_per_second": 1.306, "train/total_flos": 645966638976000.0, "train/train_loss": 1.7031736066264491}

wandb/run-20240306_145455-h1uv5tyi/logs/debug-internal.log

@@ -833,3 +833,36 @@
 2024-03-06 15:09:43,756 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
 2024-03-06 15:09:43,797 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
 2024-03-06 15:09:45,364 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:48,757 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:48,798 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:50,365 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:53,758 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:53,799 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:55,366 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:09:56,802 DEBUG   SenderThread:137 [sender.py:send():382] send: stats
+2024-03-06 15:09:58,769 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:09:58,803 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:00,367 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:02,535 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:10:02,537 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:10:02,537 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:10:02,537 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:10:03,108 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:10:03,653 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: partial_history
+2024-03-06 15:10:03,655 DEBUG   SenderThread:137 [sender.py:send():382] send: history
+2024-03-06 15:10:03,655 DEBUG   SenderThread:137 [sender.py:send_request():409] send_request: summary_record
+2024-03-06 15:10:03,656 INFO    SenderThread:137 [sender.py:_save_file():1403] saving file wandb-summary.json with policy end
+2024-03-06 15:10:03,826 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:04,108 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/wandb-summary.json
+2024-03-06 15:10:04,657 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:05,368 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:06,109 INFO    Thread-18 :137 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240306_145455-h1uv5tyi/files/output.log
+2024-03-06 15:10:08,851 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:09,658 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:10,368 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:13,851 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:14,659 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:15,369 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:18,852 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: keepalive
+2024-03-06 15:10:19,660 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report
+2024-03-06 15:10:20,370 DEBUG   HandlerThread:137 [handler.py:handle_request():146] handle_request: status_report