Training in progress - step 16000

.gitattributes

@@ -1,3 +1,4 @@
 *.safetensors filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
 tokenizer_config.json -filter -diff -merge text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

asr_modeling.py

@@ -38,7 +38,7 @@ class ASRModel(PreTrainedModel, GenerationMixin):
     _is_loading_from_pretrained: bool = False
     _pretrained_model_path: Optional[str] = None
 
-    TRANSCRIBE_PROMPT = ""
+    TRANSCRIBE_PROMPT = "Transcribe speech to text"  # Audio tokens come BEFORE this
 
     @classmethod
     def from_pretrained(cls, pretrained_model_name_or_path: str, *args, **kwargs) -> "ASRModel":
@@ -571,11 +571,10 @@ class ASRModel(PreTrainedModel, GenerationMixin):
             messages: list[dict[str, str]] = []
             if system_prompt:
                 messages.append({"role": "system", "content": system_prompt})
-            # Audio tokens only (instruction-free)
-            user_content = audio_placeholder
-            if self.TRANSCRIBE_PROMPT:
-                user_content += " " + self.TRANSCRIBE_PROMPT
-            messages.append({"role": "user", "content": user_content})
+            # Audio BEFORE prompt for proper causal attention
+            messages.append(
+                {"role": "user", "content": audio_placeholder + " " + self.TRANSCRIBE_PROMPT}
+            )
 
             chat_result = self.tokenizer.apply_chat_template(
                 messages,
@@ -654,11 +653,10 @@ class ASRModel(PreTrainedModel, GenerationMixin):
         messages: list[dict[str, str]] = []
         if system_prompt:
             messages.append({"role": "system", "content": system_prompt})
-        # Audio tokens only (instruction-free)
-        user_content = audio_placeholder
-        if self.TRANSCRIBE_PROMPT:
-            user_content += " " + self.TRANSCRIBE_PROMPT
-        messages.append({"role": "user", "content": user_content})
+        # Audio BEFORE prompt for proper causal attention
+        messages.append(
+            {"role": "user", "content": audio_placeholder + " " + self.TRANSCRIBE_PROMPT}
+        )
 
         chat_result = self.tokenizer.apply_chat_template(
             messages,

asr_processing.py

@@ -17,7 +17,7 @@ class ASRProcessor(ProcessorMixin):
     feature_extractor_class = "AutoFeatureExtractor"
     tokenizer_class = "AutoTokenizer"
     AUDIO_TOKEN = "<audio>"
-    TRANSCRIBE_PROMPT = ""
+    TRANSCRIBE_PROMPT = "Transcribe speech to text"
     # Default conv layers for Whisper/GLM-ASR: [(pad, kernel, stride), ...]
     DEFAULT_ENCODER_CONV_LAYERS = [(1, 3, 1), (1, 3, 2)]
 
@@ -89,13 +89,11 @@ class ASRProcessor(ProcessorMixin):
         else:
             num_audio_tokens = 0
 
-        # Build prompt with audio token placeholders (instruction-free)
+        # Build prompt with audio token placeholders (audio BEFORE prompt)
         if num_audio_tokens > 0:
-            user_content = self.AUDIO_TOKEN * num_audio_tokens
-            if self.TRANSCRIBE_PROMPT:
-                user_content += " " + self.TRANSCRIBE_PROMPT
+            user_content = self.AUDIO_TOKEN * num_audio_tokens + " " + self.TRANSCRIBE_PROMPT
         else:
-            user_content = self.TRANSCRIBE_PROMPT or ""
+            user_content = self.TRANSCRIBE_PROMPT
 
         messages = []
         if system_prompt: