updated

run.sh

@@ -2,18 +2,17 @@
 python run_whisper_finetuning.py \
 	--model_name_or_path="openai/whisper-small" \
 	--output_dir="../whisper-testrun1" \
-	--repo_id="NbAiLab/whisper-testrun1" \
 	--overwrite_output_dir=True \
 	--language="Norwegian" \
 	--task="transcribe" \
-	--dataset_name="NbAiLab/NPSC" \
-	--dataset_config="16K_mp3" \
+	--dataset_name="mozilla-foundation/common_voice_11_0" \
+	--dataset_config="nn-NO" \
 	--do_train=True \
 	--do_eval=True \
 	--audio_column_name="audio" \
-	--text_column_name="normsentence_text" \
-	--per_device_train_batch_size=16 \
-       	--per_device_train_batch_size=16 \
+	--text_column_name="sentence" \
+	--per_device_train_batch_size=32 \
+    --per_device_train_batch_size=32 \
 	--learning_rate=2e-5 \
 	--warmup_steps=500 \
 	--max_steps=10000 \
@@ -23,6 +22,7 @@ python run_whisper_finetuning.py \
 	--evaluation_strategy="steps" \
 	--save_steps=1000 \
 	--eval_steps=1000 \
+	--max_eval_samples=10 \
 	--logging_steps=250 \
 	--fp16=True \
 	--load_best_model_at_end=True \

run_npsc.sh

@@ -0,0 +1,37 @@
+
+python run_whisper_finetuning.py \
+	--model_name_or_path="openai/whisper-small" \
+	--output_dir="../whisper-testrun1" \
+	--overwrite_output_dir=True \
+	--language="Norwegian" \
+	--task="transcribe" \
+	--dataset_name="NbAiLab/NPSC" \
+	--dataset_config="16K_mp3_bokmaal" \
+	--do_train=True \
+	--do_eval=True \
+	--audio_column_name="audio" \
+	--text_column_name="text" \
+	--per_device_train_batch_size=16 \
+       	--per_device_train_batch_size=16 \
+	--learning_rate=2e-5 \
+	--warmup_steps=500 \
+	--max_steps=10000 \
+	--gradient_checkpointing=True \
+	--gradient_accumulation_steps=1 \
+	--group_by_length=False \
+	--evaluation_strategy="steps" \
+	--save_steps=1000 \
+	--eval_steps=1000 \
+	--logging_steps=250 \
+	--fp16=True \
+	--load_best_model_at_end=True \
+	--metric_for_best_model="wer" \
+	--greater_is_better=False \
+	--report_to="tensorboard" \
+	--predict_with_generate=True \
+	--generation_max_length=225 \
+	--print_training_arguments=True \
+	--push_to_hub=True
+
+
+	

run_nst.sh

@@ -0,0 +1,38 @@
+
+python run_whisper_finetuning.py \
+	--model_name_or_path="openai/whisper-small" \
+	--output_dir="../whisper-testrun1" \
+	--overwrite_output_dir=True \
+	--language="Norwegian" \
+	--task="transcribe" \
+	--dataset_name="NbAiLab/NST" \
+	--dataset_config="no-close" \
+	--do_train=True \
+	--do_eval=True \
+	--audio_column_name="audio" \
+	--text_column_name="text" \
+	--per_device_train_batch_size=16 \
+       	--per_device_train_batch_size=16 \
+	--learning_rate=2e-5 \
+	--warmup_steps=500 \
+	--max_steps=10000 \
+	--gradient_checkpointing=True \
+	--gradient_accumulation_steps=1 \
+	--group_by_length=False \
+	--evaluation_strategy="steps" \
+	--save_steps=1000 \
+	--eval_steps=10 \
+	--max_eval_samples=100 \
+	--logging_steps=250 \
+	--fp16=True \
+	--load_best_model_at_end=True \
+	--metric_for_best_model="wer" \
+	--greater_is_better=False \
+	--report_to="tensorboard" \
+	--predict_with_generate=True \
+	--generation_max_length=225 \
+	--print_training_arguments=True \
+	--push_to_hub=True
+
+
+	

run_whisper_finetuning.py

@@ -51,6 +51,48 @@ from transformers.utils.versions import require_version
 def list_field(default=None, metadata=None):
     return field(default_factory=lambda: default, metadata=metadata)
 
+@dataclass
+class Seq2SeqTrainingArguments(TrainingArguments):
+    """
+    Args:
+        sortish_sampler (`bool`, *optional*, defaults to `False`):
+            Whether to use a *sortish sampler* or not. Only possible if the underlying datasets are *Seq2SeqDataset*
+            for now but will become generally available in the near future.
+            It sorts the inputs according to lengths in order to minimize the padding size, with a bit of randomness
+            for the training set.
+        predict_with_generate (`bool`, *optional*, defaults to `False`):
+            Whether to use generate to calculate generative metrics (ROUGE, BLEU).
+        generation_max_length (`int`, *optional*):
+            The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default to the
+            `max_length` value of the model configuration.
+        generation_num_beams (`int`, *optional*):
+            The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default to the
+            `num_beams` value of the model configuration.
+    """
+
+    sortish_sampler: bool = field(default=False, metadata={"help": "Whether to use SortishSampler or not."})
+    predict_with_generate: bool = field(
+        default=False, metadata={"help": "Whether to use generate to calculate generative metrics (ROUGE, BLEU)."}
+    )
+    generation_max_length: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default "
+                "to the `max_length` value of the model configuration."
+            )
+        },
+    )
+    generation_num_beams: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": (
+                "The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default "
+                "to the `num_beams` value of the model configuration."
+            )
+        },
+    )
+
 
 @dataclass
 class ModelArguments:
@@ -134,6 +176,7 @@ class ModelArguments:
     )
 
 
+
 @dataclass
 class DataTrainingArguments:
     """
@@ -191,7 +234,7 @@ class DataTrainingArguments:
         default=None,
         metadata={
             "help": "For debugging purposes or quicker training, truncate the number of validation examples to this "
-                    "value if set."
+                    "value if set. Should also be set when streaming."
         },
     )
     chars_to_ignore: Optional[List[str]] = list_field(
@@ -240,19 +283,11 @@ class DataTrainingArguments:
         default="|",
         metadata={"help": "The word delimiter token for the tokenizer"},
     )
-    predict_with_generate: bool = field(
-        default=True,
-        metadata={
-            "help": "Output tokens in addition to loss and digits for calculating metrics"},
-    )
-    generation_max_length: int = field(
-        default=225,
-        metadata={"help": "Maximum number of tokens generated"},
-    )
+
     phoneme_language: Optional[str] = field(
         default=None,
         metadata={
-            "help": "The target language that should be used be"
+           "help": "The target language that should be used be"
                     " passed to the tokenizer for tokenization. Note that"
                     " this is only relevant if the model classifies the"
                     " input audio to a sequence of phoneme sequences."
@@ -303,7 +338,7 @@ def main():
     # or by passing the --help flag to this script.
     # We now keep distinct sets of args, for a cleaner separation of concerns.
     parser = HfArgumentParser(
-        (ModelArguments, DataTrainingArguments, TrainingArguments))
+        (ModelArguments, DataTrainingArguments, Seq2SeqTrainingArguments))
     model_args, data_args, training_args = parser.parse_args_into_dataclasses()
 
     # Metrics
@@ -351,7 +386,7 @@ def main():
 
     # Load dataset
     train_dataset = load_dataset(data_args.dataset_name, data_args.dataset_config_name, split="train", streaming=True, use_auth_token=True)
-    eval_dataset = load_dataset(data_args.dataset_name, data_args.dataset_config_name, split="validation", streaming=True, use_auth_token=True)
+    eval_dataset = load_dataset(data_args.dataset_name, data_args.dataset_config_name, split="test", streaming=True, use_auth_token=True)
 
     
     # Rename columns
@@ -373,15 +408,17 @@ def main():
         model_args.model_name_or_path, language=model_args.language, task=model_args.task)
     data_collator = DataCollatorSpeechSeq2SeqWithPadding(processor=processor)
 
-    
+
     # Prepare data
-    train_dataset = train_dataset.cast_column("audio", Audio(sampling_rate=16000))
-    eval_dataset = eval_dataset.cast_column("audio", Audio(sampling_rate=16000))
+    # Is not working.... but since it is already 16000 maybe I dont need it?
+    # train_dataset = train_dataset.cast_column("audio", Audio(sampling_rate=16000))
+    # eval_dataset = eval_dataset.cast_column("audio", Audio(sampling_rate=16000))
     
     # TODO Not able to implement in Streaming mode. Can not find a way to list columns. But is is necessary?
     # train_data = train_data.map(prepare_dataset, remove_columns=train_data.column_names, num_proc=1)
     
     train_dataset = train_dataset.map(prepare_dataset)
+    eval_dataset = eval_dataset.map(prepare_dataset)
     
     # Metrics
     metric = evaluate.load("wer")
@@ -407,8 +444,10 @@ def main():
 
         # use last checkpoint if exist
         if last_checkpoint is not None:
+            print("*** Found a checkpoint!")
             checkpoint = last_checkpoint
         elif os.path.isdir(model_args.model_name_or_path):
+            print("*** Loading checkpoint from parameters")
             checkpoint = model_args.model_name_or_path
         else:
             checkpoint = None
@@ -423,7 +462,13 @@ def main():
 
         # Set seed before initializing model.
         set_seed(training_args.seed)
-
+        
+        # TODO - I think the number of epochs needs to be set manually? Now it seems to be calculated based on the save steps. How do I do this?
+        # Code here
+        
+        # Save the processor as well, since we need it later
+        processor.save_pretrained(training_args.output_dir)
+        
         trainer = Seq2SeqTrainer(
             args=training_args,
             model=model,
@@ -433,6 +478,7 @@ def main():
             compute_metrics=compute_metrics,
             tokenizer=processor.feature_extractor,
         )
+        
 
         train_result = trainer.train(resume_from_checkpoint=checkpoint)
         trainer.save_model()
@@ -448,21 +494,22 @@ def main():
             trainer.create_model_card(**kwargs)
 
     # TODO - Look closer into the evaluation and the model card writing.
-
+    
+    # breakpoint()
     # Evaluation
-    results = {}
-    if training_args.do_eval:
-        logger.info("*** Evaluate ***")
-        metrics = trainer.evaluate()
-        max_eval_samples = (
-            data_args.max_eval_samples if data_args.max_eval_samples is not None else len(
-                vectorized_datasets["eval"])
-        )
-        metrics["eval_samples"] = min(
-            max_eval_samples, len(vectorized_datasets["eval"]))
-
-        trainer.log_metrics("eval", metrics)
-        trainer.save_metrics("eval", metrics)
+    # results = {}
+    # if training_args.do_eval:
+    #     logger.info("*** Evaluate ***")
+    #     metrics = trainer.evaluate()
+    #     max_eval_samples = (
+    #         data_args.max_eval_samples if data_args.max_eval_samples is not None else len(
+    #             vectorized_datasets["eval"])
+    #     )
+    #     metrics["eval_samples"] = min(
+    #         max_eval_samples, len(vectorized_datasets["eval"]))
+
+    #     trainer.log_metrics("eval", metrics)
+    #     trainer.save_metrics("eval", metrics)
 
     # Write model card and (optionally) push to hub
     config_name = data_args.dataset_config_name if data_args.dataset_config_name is not None else "na"