Model save

README.md

@@ -1,6 +1,6 @@
 ---
 license: cc-by-nc-4.0
-base_model: facebook/mms-1b-all
+base_model: facebook/mms-1b-fl102
 tags:
 - generated_from_trainer
 datasets:
@@ -15,7 +15,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # breeze-listen-w2v2-kn-GF
 
-This model is a fine-tuned version of [facebook/mms-1b-all](https://huggingface.co/facebook/mms-1b-all) on the fleurs dataset.
+This model is a fine-tuned version of [facebook/mms-1b-fl102](https://huggingface.co/facebook/mms-1b-fl102) on the fleurs dataset.
 
 ## Model description
 
@@ -35,12 +35,12 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.001
-- train_batch_size: 1
+- train_batch_size: 4
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
-- gradient_accumulation_steps: 32
-- total_train_batch_size: 32
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
 - lr_scheduler_warmup_steps: 100

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.99,
+    "train_loss": 3.364711216517857,
+    "train_runtime": 13678.922,
+    "train_samples": 2471,
+    "train_samples_per_second": 0.723,
+    "train_steps_per_second": 0.023
+}

breeze-listen-w2v2-kn-GF.log

@@ -1,5 +1,5 @@
-02/04/2024 13:54:35 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, 16-bits training: True
-02/04/2024 13:54:35 - INFO - __main__ - Training/evaluation parameters TrainingArguments(
+02/04/2024 18:56:48 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, 16-bits training: True
+02/04/2024 18:56:48 - INFO - __main__ - Training/evaluation parameters TrainingArguments(
 _n_gpu=1,
 adafactor=False,
 adam_beta1=0.9,
@@ -39,7 +39,7 @@ fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},
 fsdp_min_num_params=0,
 fsdp_transformer_layer_cls_to_wrap=None,
 full_determinism=False,
-gradient_accumulation_steps=32,
+gradient_accumulation_steps=16,
 gradient_checkpointing=True,
 gradient_checkpointing_kwargs=None,
 greater_is_better=None,
@@ -64,7 +64,7 @@ local_rank=0,
 log_level=passive,
 log_level_replica=warning,
 log_on_each_node=True,
-logging_dir=/cosmos/home/sp-operator/ai/training/models/simpragma/breeze-listen-w2v2-kn-GF/runs/Feb04_13-54-35_knight,
+logging_dir=/cosmos/home/sp-operator/ai/training/models/simpragma/breeze-listen-w2v2-kn-GF/runs/Feb04_18-56-48_knight,
 logging_first_step=False,
 logging_nan_inf_filter=True,
 logging_steps=500,
@@ -84,7 +84,7 @@ output_dir=/cosmos/home/sp-operator/ai/training/models/simpragma/breeze-listen-w
 overwrite_output_dir=True,
 past_index=-1,
 per_device_eval_batch_size=8,
-per_device_train_batch_size=1,
+per_device_train_batch_size=4,
 prediction_loss_only=False,
 push_to_hub=True,
 push_to_hub_model_id=None,
@@ -119,4 +119,4 @@ warmup_ratio=0.0,
 warmup_steps=100,
 weight_decay=0.0,
 )
-{'train_runtime': 13678.922, 'train_samples_per_second': 0.723, 'train_steps_per_second': 0.023, 'train_loss': 3.364711216517857, 'epoch': 3.99}
+{'train_runtime': 12199.309, 'train_samples_per_second': 0.81, 'train_steps_per_second': 0.012, 'train_loss': 3.1379870364540503, 'epoch': 3.94}

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "facebook/mms-1b-all",
+  "_name_or_path": "facebook/mms-1b-fl102",
   "activation_dropout": 0.05,
   "adapter_attn_dim": 16,
   "adapter_kernel_size": 3,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84fe74380883a43f5a315e535ef7668f1661b11fa85249c1f5ea0efd5b201db3
+oid sha256:c5019ad31159ea5df95db8779bd8072d6f2dfcaa73fce699f4d1ef7fdd5b3043
 size 3859264976

train-ctc-model.sh

@@ -55,8 +55,8 @@ echo "OUTDIR: ${OUTDIR}"
 # Training parameters you can tweak. Feel free to directly change any of the parameters below.
 
 MAX_EPOCHS=4
-TRAIN_BATCH_SIZE=2
-EVAL_BATCH_SIZE=2
+TRAIN_BATCH_SIZE=1
+EVAL_BATCH_SIZE=1
 LEARNING_RATE="1e-3"
 
 EVAL_STEPS="1000"
@@ -80,6 +80,7 @@ python ${SCRIPT_DIR}/run_speech_recognition_ctc_adapter.py \
 	--output_dir="${OUTDIR}" \
 	--num_train_epochs="${MAX_EPOCHS}" \
 	--per_device_train_batch_size="${TRAIN_BATCH_SIZE}" \
+	--gradient_accumulation_steps="32"	\
 	--learning_rate="${LEARNING_RATE}" \
 	--warmup_steps="100" \
 	--evaluation_strategy="steps" \

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.99,
+    "train_loss": 3.364711216517857,
+    "train_runtime": 13678.922,
+    "train_samples": 2471,
+    "train_samples_per_second": 0.723,
+    "train_steps_per_second": 0.023
+}

trainer_state.json

@@ -0,0 +1,30 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.9886685552407934,
+  "eval_steps": 1000,
+  "global_step": 308,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 3.99,
+      "step": 308,
+      "total_flos": 1.1188723782736937e+19,
+      "train_loss": 3.364711216517857,
+      "train_runtime": 13678.922,
+      "train_samples_per_second": 0.723,
+      "train_steps_per_second": 0.023
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 308,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 1000,
+  "total_flos": 1.1188723782736937e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:889214cde0e59492b6da27c312fdf4c9aa1ba5439a639a7b5f6e3a9dc0f91c73
+oid sha256:4aaa9085f71e66c7740edfe87fb1bf5ecbe420672688ea0bc427245f20a7e66a
 size 4856