Training in progress - step 29500

.gitattributes

@@ -1,3 +1,4 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 tokenizer_config.json -filter -diff -merge text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

asr_config.py

@@ -67,7 +67,7 @@ class ASRConfig(transformers.PretrainedConfig):
         # Set default generation parameters (greedy decoding only)
         generation_defaults = {
             "num_beams": 1,
-            "max_new_tokens": 128,
+            "max_new_tokens": 256,
             "min_new_tokens": 0,
             "repetition_penalty": 1.0,
             "length_penalty": 1.0,

asr_pipeline.py

@@ -496,30 +496,5 @@ class ASRPipeline(transformers.AutomaticSpeechRecognitionPipeline):
         # Convert "eur X" to "X euros" for Whisper normalizer compatibility
         text = re.sub(r"\beur\s+(\d+)", r"\1 euros", text)
 
-        # 4. TRUNCATE TRAILING REPEATS
-        text = self._truncate_trailing_repeats(text)
-
-        # 5. STRIP WHITESPACE
+        # 4. STRIP WHITESPACE
         return re.sub(r"\s+", " ", text).strip()
-
-    def _truncate_trailing_repeats(self, text: str, max_ngram: int = 4) -> str:
-        """Remove trailing repeated n-grams (1-4 words)."""
-        words = text.split()
-        if len(words) < 2:
-            return text
-
-        # Keep truncating until no more trailing repeats found
-        changed = True
-        while changed:
-            changed = False
-            # Check for repeating n-grams from largest to smallest
-            for n in range(min(max_ngram, len(words) // 2), 0, -1):
-                if len(words) < n * 2:
-                    continue
-                # Check if last n words repeat the previous n words
-                if words[-n:] == words[-2 * n : -n]:
-                    words = words[:-n]  # Remove the trailing repeat
-                    changed = True
-                    break  # Restart from largest n-gram
-
-        return " ".join(words)