Added updated training script.

run_speech_recognition_seq2seq_streaming.py

@@ -479,6 +479,7 @@ def main():
     FULL2HALF = dict((i + 0xFEE0, i) for i in range(0x21, 0x7F))
     FULL2HALF[0x3000] = 0x20
 
+
     def prepare_dataset(batch):
         # process audio
         sample = batch[audio_column_name]
@@ -491,18 +492,20 @@ def main():
         input_str = batch[text_column_name].lower() if do_lower_case else batch[text_column_name]
         if do_remove_punctuation:
             input_str = normalizer(input_str).strip()
+        batch['labels'] = input_str.translate(FULL2HALF)
+
+        return batch
 
-        input_str = input_str.translate(FULL2HALF)
-        input_str = tagger.parse(input_str)
 
+    def tokenize_labels(batch):
+        input_str = tagger.parse(batch['labels'])
         batch["labels"] = tokenizer(input_str).input_ids
         return batch
 
+
     with training_args.main_process_first(desc="dataset map pre-processing"):
-        vectorized_datasets = raw_datasets.map(
-            prepare_dataset,
-            remove_columns=raw_datasets_features,
-        ).with_format("torch")
+        prep_datasets = raw_datasets.map(prepare_dataset, remove_columns=raw_datasets_features)
+        vectorized_datasets = prep_datasets.map(tokenize_labels).with_format("torch")
 
         if training_args.do_train and data_args.streaming:
             # manually shuffle if streaming (done by the trainer for non-streaming)