Training in progress, epoch 2

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1f7da727958b65542249e558b0eb2f6eb4d3c54f857853c697e21464087694cc
+oid sha256:29a6328c0e5235532f4cbbbf0828e74f77acd14fdd518a79a69199537d11f533
 size 502675828

runs/Apr10_15-06-31_9088970cc926/events.out.tfevents.1712761592.9088970cc926.34.1

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1ae3b5b714ffae5c3d4f2e003e44a80936fc088a063b7d6e54d640eabc907ffa
-size 5845
+oid sha256:a390005918cd3baf7ed639de34b8d335858c28b7a4e71cf79f2290cba6fe976b
+size 6528

wandb/debug-internal.log

@@ -322,3 +322,33 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-10 15:08:11,983 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
 2024-04-10 15:08:11,997 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
 2024-04-10 15:08:11,998 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:08:14,264 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:19,265 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:24,266 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:26,983 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:08:26,997 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:08:26,998 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:08:29,403 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:08:29,404 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:08:29,404 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:08:29,405 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:29,405 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:08:29,811 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:08:30,831 DEBUG   SenderThread:100 [sender.py:send():379] send: stats
+2024-04-10 15:08:32,249 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:08:32,251 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:08:32,251 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:08:32,252 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:08:32,813 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:08:34,814 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log
+2024-04-10 15:08:35,258 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:40,259 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:41,983 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:08:41,997 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:08:41,998 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:08:46,244 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:51,245 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:56,246 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:56,983 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:08:56,998 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:08:56,998 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status

wandb/run-20240410_150123-cuua991r/files/output.log

@@ -11,4 +11,6 @@ You should probably TRAIN this model on a down-stream task to be able to use it
 /opt/conda/lib/python3.10/site-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches', 'even_batches', 'use_seedable_sampler']). Please pass an `accelerate.DataLoaderConfiguration` instead:
 dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False, even_batches=True, use_seedable_sampler=True)
   warnings.warn(
+/opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
+  _warn_prf(average, modifier, msg_start, len(result))
 /opt/conda/lib/python3.10/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.

wandb/run-20240410_150123-cuua991r/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.9466, "train/grad_norm": 6.16647481918335, "train/learning_rate": 9.008474576271187e-06, "train/epoch": 1.0, "train/global_step": 118, "_timestamp": 1712761649.556628, "_runtime": 366.0058960914612, "_step": 9, "eval/loss": 1.884623408317566, "eval/accuracy": 0.3028169014084507, "eval/precision": 0.21421540500035863, "eval/recall": 0.3028169014084507, "eval/f1": 0.20265131813759552, "eval/runtime": 2.8549, "eval/samples_per_second": 49.739, "eval/steps_per_second": 6.305}
+{"train/loss": 1.878, "train/grad_norm": 8.734905242919922, "train/learning_rate": 8.008474576271187e-06, "train/epoch": 2.0, "train/global_step": 236, "_timestamp": 1712761712.249299, "_runtime": 428.6985671520233, "_step": 11, "eval/loss": 1.871309757232666, "eval/accuracy": 0.2676056338028169, "eval/precision": 0.1656163852860285, "eval/recall": 0.2676056338028169, "eval/f1": 0.17064762813956771, "eval/runtime": 2.8419, "eval/samples_per_second": 49.967, "eval/steps_per_second": 6.334}

wandb/run-20240410_150123-cuua991r/logs/debug-internal.log

@@ -322,3 +322,33 @@ subprocess.TimeoutExpired: Command '['conda', 'env', 'export']' timed out after
 2024-04-10 15:08:11,983 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
 2024-04-10 15:08:11,997 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
 2024-04-10 15:08:11,998 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:08:14,264 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:19,265 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:24,266 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:26,983 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:08:26,997 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:08:26,998 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:08:29,403 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:08:29,404 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:08:29,404 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:08:29,405 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:29,405 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:08:29,811 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:08:30,831 DEBUG   SenderThread:100 [sender.py:send():379] send: stats
+2024-04-10 15:08:32,249 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: partial_history
+2024-04-10 15:08:32,251 DEBUG   SenderThread:100 [sender.py:send():379] send: history
+2024-04-10 15:08:32,251 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: summary_record
+2024-04-10 15:08:32,252 INFO    SenderThread:100 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end
+2024-04-10 15:08:32,813 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/wandb-summary.json
+2024-04-10 15:08:34,814 INFO    Thread-12 :100 [dir_watcher.py:_on_file_modified():288] file/dir modified: /kaggle/working/wandb/run-20240410_150123-cuua991r/files/output.log
+2024-04-10 15:08:35,258 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:40,259 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:41,983 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:08:41,997 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:08:41,998 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status
+2024-04-10 15:08:46,244 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:51,245 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:56,246 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: status_report
+2024-04-10 15:08:56,983 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: internal_messages
+2024-04-10 15:08:56,998 DEBUG   HandlerThread:100 [handler.py:handle_request():146] handle_request: stop_status
+2024-04-10 15:08:56,998 DEBUG   SenderThread:100 [sender.py:send_request():406] send_request: stop_status