fix: enhance audio processing in transcribe function to handle tuple input and resample to 16kHz

app.py

@@ -34,30 +34,26 @@ def transcribe(audio, state=""):
         # Try extracting the first element
         audio = audio[1] if len(audio) > 1 else None
         
-    if isinstance(audio, np.ndarray):
-        audio_buffer.append(audio)
-        # Process if buffer has enough data (e.g., 5 seconds at 16kHz)
-        if len(np.concatenate(audio_buffer)) >= 5 * 16000:
-            # Concatenate and preprocess
-            audio_data = np.concatenate(audio_buffer)
-            audio_data = audio_data.mean(axis=1) if audio_data.ndim > 1 else audio_data  # To mono
-            temp_file = "temp_audio.wav"
-            sf.write(temp_file, audio_data, samplerate=16000)
-            print("Transcribing audio...")
-
-            # Transcribe
-            if torch.cuda.is_available():
-                model = model.cuda()
-            transcription = model.transcribe([temp_file])[0]
-            print(f"Transcription: {transcription}")
-            model = model.cpu()
-            os.remove(temp_file)
-            print("Temporary file removed.")
+    if isinstance(audio, tuple) and len(audio) == 2 and isinstance(audio[1], np.ndarray):
+        # Handle tuple of (sample_rate, audio_array)
+        print(f"Tuple contents: {audio}")
+        sample_rate, audio_data = audio
+        # Resample to 16kHz for NeMo
+        if sample_rate != 16000:
+            print(f"Resampling from {sample_rate}Hz to 16000Hz")
+            audio_data = librosa.resample(audio_data.astype(float), orig_sr=sample_rate, target_sr=16000)
+        # Save to temporary WAV file
+        temp_file = "temp_audio.wav"
+        sf.write(temp_file, audio_data, samplerate=16000)
+        print(f"Processing temporary audio file: {temp_file}")
+        transcription = model.transcribe([temp_file])[0]
+        os.remove(temp_file)  # Clean up
+        print("Temporary file removed.")
 
-            # Clear buffer
-            audio_buffer = []
-            new_state = state + " " + transcription if state else transcription
-            return new_state, new_state
+        # Clear buffer
+        audio_buffer = []
+        new_state = state + " " + transcription if state else transcription
+        return new_state, new_state
     return state, state
 
 # Define the Gradio interface

requirements.txt

@@ -4,4 +4,5 @@ nemo_toolkit[asr]>=1.18.0
 omegaconf>=2.2.0
 numpy>=1.22.0
 cuda-python>=12.3
-soundfile
+soundfile
+librosa