Training in progress, step 1000

README.md

@@ -11,14 +11,14 @@ model-index:
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/priyanshipal/huggingface/runs/hvo6b5jz)
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="200" height="32"/>](https://wandb.ai/priyanshipal/huggingface/runs/ovuq4xne)
 # output
 
 This model was trained from scratch on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.9947
-- Cer: 0.4133
-- Wer: 0.6195
+- Loss: 2.0007
+- Cer: 0.3962
+- Wer: 0.6069
 
 ## Model description
 
@@ -45,20 +45,49 @@ The following hyperparameters were used during training:
 - total_train_batch_size: 32
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- lr_scheduler_warmup_steps: 2000
-- training_steps: 6000
+- lr_scheduler_warmup_steps: 1500
+- training_steps: 3500
 - mixed_precision_training: Native AMP
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss | Cer    | Wer    |
-|:-------------:|:-----:|:----:|:---------------:|:------:|:------:|
-| 1.5851        | 1.6   | 1000 | 1.9947          | 0.4133 | 0.6195 |
-| 1.8352        | 3.2   | 2000 | 2.1491          | 0.4724 | 0.7895 |
-| 2.3755        | 4.8   | 3000 | 2.3793          | 0.4433 | 0.7270 |
-| 3.3134        | 6.4   | 4000 | 3.3025          | 0.5204 | 0.8033 |
-| 3.4098        | 8.0   | 5000 | 3.2885          | 0.5196 | 0.8050 |
-| 3.1155        | 9.6   | 6000 | 3.2885          | 0.5196 | 0.8050 |
+| Training Loss | Epoch  | Step | Validation Loss | Cer    | Wer    |
+|:-------------:|:------:|:----:|:---------------:|:------:|:------:|
+| 5.2909        | 0.16   | 100  | 5.7883          | 0.9933 | 1.0    |
+| 4.4471        | 0.32   | 200  | 3.7597          | 0.9812 | 1.0    |
+| 3.6763        | 0.48   | 300  | 3.7718          | 0.9769 | 1.0    |
+| 3.1272        | 0.64   | 400  | 3.4647          | 0.9826 | 1.0    |
+| 3.0699        | 0.8    | 500  | 2.7480          | 0.6002 | 0.8337 |
+| 1.7389        | 0.96   | 600  | 2.4468          | 0.4717 | 0.6782 |
+| 1.7013        | 1.12   | 700  | 2.1151          | 0.4489 | 0.6820 |
+| 1.2995        | 1.28   | 800  | 2.0757          | 0.4160 | 0.6245 |
+| 1.6852        | 1.44   | 900  | 1.9870          | 0.4112 | 0.6154 |
+| 1.3997        | 1.6    | 1000 | 2.0007          | 0.3962 | 0.6069 |
+| 1.768         | 1.76   | 1100 | 2.0712          | 0.4123 | 0.6448 |
+| 2.5192        | 1.92   | 1200 | 2.5729          | 0.6884 | 0.9178 |
+| 2.6077        | 2.08   | 1300 | 2.4078          | 0.4816 | 0.8066 |
+| 2.6928        | 2.24   | 1400 | 2.3596          | 0.4904 | 0.7915 |
+| 0.0           | 2.4    | 1500 | 2.4471          | 0.6019 | 0.8782 |
+| 0.0           | 2.56   | 1600 | 2.4490          | 0.6112 | 0.8888 |
+| 0.0           | 2.7200 | 1700 | nan             | 1.0    | 1.0    |
+| 0.0           | 2.88   | 1800 | nan             | 1.0    | 1.0    |
+| 0.0           | 3.04   | 1900 | nan             | 1.0    | 1.0    |
+| 0.0           | 3.2    | 2000 | nan             | 1.0    | 1.0    |
+| 0.0           | 3.36   | 2100 | nan             | 1.0    | 1.0    |
+| 0.0           | 3.52   | 2200 | nan             | 1.0    | 1.0    |
+| 0.0           | 3.68   | 2300 | nan             | 1.0    | 1.0    |
+| 0.0           | 3.84   | 2400 | nan             | 1.0    | 1.0    |
+| 0.0           | 4.0    | 2500 | nan             | 1.0    | 1.0    |
+| 0.0           | 4.16   | 2600 | nan             | 1.0    | 1.0    |
+| 0.0           | 4.32   | 2700 | nan             | 1.0    | 1.0    |
+| 0.0           | 4.48   | 2800 | nan             | 1.0    | 1.0    |
+| 0.0           | 4.64   | 2900 | nan             | 1.0    | 1.0    |
+| 0.0           | 4.8    | 3000 | nan             | 1.0    | 1.0    |
+| 0.0           | 4.96   | 3100 | nan             | 1.0    | 1.0    |
+| 0.0           | 5.12   | 3200 | nan             | 1.0    | 1.0    |
+| 0.0           | 5.28   | 3300 | nan             | 1.0    | 1.0    |
+| 0.0           | 5.44   | 3400 | nan             | 1.0    | 1.0    |
+| 0.0           | 5.6    | 3500 | nan             | 1.0    | 1.0    |
 
 
 ### Framework versions

all_results.json

@@ -1,16 +1,16 @@
 {
-    "epoch": 9.6,
-    "eval_cer": 0.4133216406903974,
-    "eval_loss": 1.9947007894515991,
-    "eval_runtime": 158.2803,
+    "epoch": 5.6,
+    "eval_cer": 0.39621716426834613,
+    "eval_loss": 2.00065016746521,
+    "eval_runtime": 157.5611,
     "eval_samples": 3136,
-    "eval_samples_per_second": 19.813,
-    "eval_steps_per_second": 1.238,
-    "eval_wer": 0.6194798466480157,
-    "total_flos": 3.700768773245485e+19,
-    "train_loss": 2.825458660195271,
-    "train_runtime": 12931.7591,
+    "eval_samples_per_second": 19.903,
+    "eval_steps_per_second": 1.244,
+    "eval_wer": 0.6068801160501502,
+    "total_flos": 2.158464150901847e+19,
+    "train_loss": 1.3790143143790108,
+    "train_runtime": 12492.843,
     "train_samples": 20000,
-    "train_samples_per_second": 14.847,
-    "train_steps_per_second": 0.464
+    "train_samples_per_second": 8.965,
+    "train_steps_per_second": 0.28
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/m/triton/scratch/elec/puhe/p/palp3/MUCS/indicwav2vec-hindi",
+  "_name_or_path": "/scratch/elec/puhe/p/palp3/MUCS/mucs_language_segregated_data/trainwithtags_warmup1500_s300_shuff100/output",
   "activation_dropout": 0.0,
   "adapter_attn_dim": null,
   "adapter_kernel_size": 3,

eval_results.json

@@ -1,10 +1,10 @@
 {
-    "epoch": 9.6,
-    "eval_cer": 0.4133216406903974,
-    "eval_loss": 1.9947007894515991,
-    "eval_runtime": 158.2803,
+    "epoch": 5.6,
+    "eval_cer": 0.39621716426834613,
+    "eval_loss": 2.00065016746521,
+    "eval_runtime": 157.5611,
     "eval_samples": 3136,
-    "eval_samples_per_second": 19.813,
-    "eval_steps_per_second": 1.238,
-    "eval_wer": 0.6194798466480157
+    "eval_samples_per_second": 19.903,
+    "eval_steps_per_second": 1.244,
+    "eval_wer": 0.6068801160501502
 }

indicwav2vec_trainwtags_MUCS_warmup1500_s300shuff100_2212167.out

@@ -12466,3 +12466,40 @@ last prediction string cs> लत/s> द्वरा अनुवादित
   eval_steps_per_second   =      1.244
   eval_wer                =     0.6069
 
+wandb: - 0.005 MB of 0.005 MB uploaded
+wandb: Run history:
+wandb:                eval/cer ████▃▂▂▁▁▁▁▄▂▂▃▃███████████████████▁
+wandb:               eval/loss █▄▄▄▂▂▁▁▁▁▁▂▂▂▂▂                   ▁
+wandb:            eval/runtime ▃▂▃▂▃▃▃▃▂▄▂▃▃▄▃▃█▃▅▄▆▂▃▃▃▅▄▃▁▅▁▂▁▁▂▂
+wandb: eval/samples_per_second ▆▇▆▇▆▆▆▆▇▅▇▆▆▅▅▆▁▆▄▅▃▇▆▆▆▄▄▆█▄█▇██▇▇
+wandb:   eval/steps_per_second ▆▇▆▇▆▆▆▆▇▅▇▆▆▅▅▆▁▆▄▅▃▇▆▆▆▄▅▆█▄█▇██▇▇
+wandb:                eval/wer ████▅▂▂▁▁▁▂▇▅▄▆▆███████████████████▁
+wandb:             train/epoch ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███
+wandb:       train/global_step ▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███
+wandb:         train/grad_norm ▄█▁▂▁▁▁▁▂▁▁▂▂▂▁▂▁                       
+wandb:     train/learning_rate ▁▁▂▂▃▃▄▄▄▅▅▆▆▇▇▇████████████████████████
+wandb:              train/loss █▄▃▃▃▃▂▂▂▂▂▂▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
+wandb: 
+wandb: Run summary:
+wandb:                 eval/cer 0.39622
+wandb:                eval/loss 2.00065
+wandb:             eval/runtime 157.5611
+wandb:  eval/samples_per_second 19.903
+wandb:    eval/steps_per_second 1.244
+wandb:                 eval/wer 0.60688
+wandb:               total_flos 2.158464150901847e+19
+wandb:              train/epoch 5.6
+wandb:        train/global_step 3500
+wandb:          train/grad_norm nan
+wandb:      train/learning_rate 0.00058
+wandb:               train/loss 0.0
+wandb:               train_loss 1.37901
+wandb:            train_runtime 12492.843
+wandb: train_samples_per_second 8.965
+wandb:   train_steps_per_second 0.28
+wandb: 
+wandb: 🚀 View run retrainwithtags_indicw2v_ad0_3_hd_02_featd_0_3_lr6e-4_warmup500_s300_shuff100 at: https://wandb.ai/priyanshipal/huggingface/runs/ovuq4xne
+wandb: ⭐️ View project at: https://wandb.ai/priyanshipal/huggingface
+wandb: Synced 6 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)
+wandb: Find logs at: ./wandb/run-20240828_021854-ovuq4xne/logs
+wandb: WARNING The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require("core")`! See https://wandb.me/wandb-core for more information.

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11123f516d513f049ccf8eca7a64f45fe94183f0c16009563846a3cba3ca71e4
+oid sha256:3a8e203801869233cda2ae2613a1891ea5c6088e160873058465ebad871dc4fc
 size 1262426580

predictionswtags_indicw2v_ad0_3_hd_02_featd_0_2_lr6e-4_warmup1500_s300_shuff100.txt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f6c01a489567285af2c593b021150526769cd8fc6436fbec716d2be55370ac92
-size 23608120
+oid sha256:4f2ea0c6efc6097550d1ac201b87b115838d77f1d3da07f714a606e3dea70192
+size 24702594

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 9.6,
-    "total_flos": 3.700768773245485e+19,
-    "train_loss": 2.825458660195271,
-    "train_runtime": 12931.7591,
+    "epoch": 5.6,
+    "total_flos": 2.158464150901847e+19,
+    "train_loss": 1.3790143143790108,
+    "train_runtime": 12492.843,
     "train_samples": 20000,
-    "train_samples_per_second": 14.847,
-    "train_steps_per_second": 0.464
+    "train_samples_per_second": 8.965,
+    "train_steps_per_second": 0.28
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83f52cedbc94f4a9afc031c9289677879e1c4f30bc20bbb9df0588e106379758
+oid sha256:225187752662e438525246e50709b06bc314c78e599731793e3282deaa2b7361
 size 5496