Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+stt_ar_fastconformer_hybrid_large_pcd_v1.0.nemo filter=lfs diff=lfs merge=lfs -text
+whisper_checkpoints/models--openai--whisper-large-v2/blobs/57a1ba2a82c093cabff2541409ae778c97145378b9ddfa722763cb1cb8f9020b filter=lfs diff=lfs merge=lfs -text

aqib-nemo-asr.py

@@ -0,0 +1,601 @@
+import asyncio
+import websockets
+import json
+import threading
+import numpy as np
+import logging
+import time
+import tempfile
+import os
+import re
+from concurrent.futures import ThreadPoolExecutor
+import nemo.collections.asr as nemo_asr
+import soundfile as sf
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+# ===== Arabic number mapping (expanded) =====
+arabic_numbers = {
+    # Basic digits
+    "صفر": "0", "زيرو": "0", "٠": "0","زيو": "0","زير": "0",
+    "واحد": "1", "واحدة": "1", "١": "1",
+    "اتنين": "2", "اثنين": "2", "إثنين": "2", "اثنان": "2", "إثنان": "2", "٢": "2",
+    "تلاتة": "3", "ثلاثة": "3", "٣": "3",
+    "اربعة": "4", "أربعة": "4", "٤": "4",
+    "خمسة": "5", "٥": "5",
+    "ستة": "6", "٦": "6",
+    "سبعة": "7", "٧": "7",
+    "تمانية": "8", "ثمانية": "8", "٨": "8",
+    "تسعة": "9", "٩": "9",
+
+    # Teens
+    "عشرة": "10", "١٠": "10",
+    "حداشر": "11", "احد عشر": "11","احداشر": "11",
+    "اتناشر": "12", "اثنا عشر": "12",
+    "تلتاشر": "13", "ثلاثة عشر": "13",
+    "اربعتاشر": "14", "أربعة عشر": "14",
+    "خمستاشر": "15", "خمسة عشر": "15",
+    "ستاشر": "16", "ستة عشر": "16",
+    "سبعتاشر": "17", "سبعة عشر": "17",
+    "طمنتاشر": "18", "ثمانية عشر": "18",
+    "تسعتاشر": "19", "تسعة عشر": "19",
+
+    # Tens
+    "عشرين": "20", "٢٠": "20",
+    "تلاتين": "30", "ثلاثين": "30", "٣٠": "30",
+    "اربعين": "40", "أربعين": "40", "٤٠": "40",
+    "خمسين": "50", "٥٠": "50",
+    "ستين": "60", "٦٠": "60",
+    "سبعين": "70", "٧٠": "70",
+    "تمانين": "80", "ثمانين": "80", "٨٠": "80","تمانون": "80","ثمانون": "80",
+    "تسعين": "90", "٩٠": "90",
+
+    # Hundreds
+    "مية": "100", "مائة": "100", "مئة": "100", "١٠٠": "100",
+    "ميتين": "200", "مائتين": "200",
+    "تلاتمية": "300", "ثلاثمائة": "300",
+    "اربعمية": "400", "أربعمائة": "400",
+    "خمسمية": "500", "خمسمائة": "500",
+    "ستمية": "600", "ستمائة": "600",
+    "سبعمية": "700", "سبعمائة": "700",
+    "تمانمية": "800", "ثمانمائة": "800",
+    "تسعمية": "900", "تسعمائة": "900",
+
+    # Thousands
+    "ألف": "1000", "الف": "1000", "١٠٠٠": "1000",
+    "ألفين": "2000", "الفين": "2000",
+    "تلات تلاف": "3000", "ثلاثة آلاف": "3000",
+    "اربعة آلاف": "4000", "أربعة آلاف": "4000",
+    "خمسة آلاف": "5000",
+    "ستة آلاف": "6000",
+    "سبعة آلاف": "7000",
+    "تمانية آلاف": "8000", "ثمانية آلاف": "8000",
+    "تسعة آلاف": "9000",
+
+    # Large numbers
+    "عشرة آلاف": "10000",
+    "مية ألف": "100000", "مائة ألف": "100000",
+    "مليون": "1000000", "١٠٠٠٠٠٠": "1000000",
+    "ملايين": "1000000",
+    "مليار": "1000000000", "١٠٠٠٠٠٠٠٠٠": "1000000000"
+}
+
+def replace_arabic_numbers(text: str) -> str:
+    for word, digit in arabic_numbers.items():
+        text = re.sub(rf"\b{word}\b", digit, text)
+    return text
+
+
+# Global NeMo model
+asr_model = None
+
+def initialize_nemo_model():
+    """Initialize NeMo FastConformer model"""
+    global asr_model
+    
+    logger.info("Loading NeMo FastConformer Arabic ASR model...")
+    
+    # Model path - adjust this to your model location
+    model_path = os.getenv(
+        "NEMO_MODEL_PATH", 
+        "/path/to/stt_ar_fastconformer_hybrid_large_pcd_v1.0.nemo"  # Update this path
+    )
+    
+    if not os.path.exists(model_path):
+        logger.error(f"Model not found at: {model_path}")
+        logger.info("Please download the model from: https://catalog.ngc.nvidia.com/orgs/nvidia/teams/nemo/models/stt_ar_fastconformer_hybrid_large_pcd")
+        raise FileNotFoundError(f"NeMo model not found: {model_path}")
+    
+    try:
+        asr_model = nemo_asr.models.EncDecCTCModel.restore_from(model_path)
+        logger.info("NeMo FastConformer model loaded successfully")
+        
+        # Set model to eval mode for inference
+        asr_model.eval()
+        
+    except Exception as e:
+        logger.error(f"Failed to load NeMo model: {e}")
+        raise
+
+# Initialize model on startup
+initialize_nemo_model()
+
+# Thread pool for processing
+executor = ThreadPoolExecutor(max_workers=4)
+
+class JambonzAudioBuffer:
+    def __init__(self, sample_rate=8000, chunk_duration=1.0):
+        self.sample_rate = sample_rate
+        self.chunk_duration = chunk_duration
+        self.chunk_samples = int(chunk_duration * sample_rate)
+        
+        self.buffer = np.array([], dtype=np.float32)
+        self.lock = threading.Lock()
+        self.total_audio = np.array([], dtype=np.float32)
+        
+        # Voice Activity Detection
+        self.silence_threshold = 0.05
+        self.min_speech_samples = int(0.5 * sample_rate)
+        
+    def add_audio(self, audio_data):
+        with self.lock:
+            self.buffer = np.concatenate([self.buffer, audio_data])
+            self.total_audio = np.concatenate([self.total_audio, audio_data])
+    
+    def has_chunk_ready(self):
+        with self.lock:
+            return len(self.buffer) >= self.chunk_samples
+    
+    def is_speech(self, audio_chunk):
+        """Simple VAD based on energy"""
+        if len(audio_chunk) < self.min_speech_samples:
+            return False
+        energy = np.mean(np.abs(audio_chunk))
+        return energy > self.silence_threshold
+    
+    def get_chunk_for_processing(self):
+        """Get audio chunk for processing"""
+        with self.lock:
+            if len(self.buffer) < self.chunk_samples:
+                return None
+            return np.array([1])  # Signal that chunk is ready
+    
+    def get_all_audio(self):
+        """Get all accumulated audio"""
+        with self.lock:
+            return self.total_audio.copy()
+    
+    def clear(self):
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+    
+    def reset_for_new_segment(self):
+        """Reset buffers for new transcription segment"""
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+
+def linear16_to_audio(audio_bytes, sample_rate=8000):
+    """Convert LINEAR16 PCM bytes to numpy array"""
+    try:
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16)
+        audio_array = audio_array.astype(np.float32) / 32768.0
+        return audio_array
+    except Exception as e:
+        logger.error(f"Error converting LINEAR16 to audio: {e}")
+        return np.array([], dtype=np.float32)
+
+def resample_audio(audio_data, source_rate, target_rate):
+    """Resample audio to target sample rate"""
+    if source_rate == target_rate:
+        return audio_data
+    
+    if source_rate == 8000 and target_rate == 16000:
+        # Simple 2x upsampling for common case
+        upsampled = np.repeat(audio_data, 2)
+        return upsampled.astype(np.float32)
+    
+    # Fallback: Linear interpolation resampling
+    ratio = target_rate / source_rate
+    indices = np.arange(0, len(audio_data), 1/ratio)
+    indices = indices[indices < len(audio_data)]
+    resampled = np.interp(indices, np.arange(len(audio_data)), audio_data)
+    
+    return resampled.astype(np.float32)
+
+def transcribe_with_nemo(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio using NeMo FastConformer"""
+    try:
+        if len(audio_data) == 0:
+            return ""
+        
+        # Resample to 16kHz (NeMo models typically expect 16kHz)
+        resampled_audio = resample_audio(audio_data, 8000, 16000)
+        
+        # Skip very short audio
+        min_samples = int(0.3 * 16000)
+        if len(resampled_audio) < min_samples:
+            return ""
+        
+        start_time = time.time()
+        
+        # Save audio to temporary file (NeMo expects file path)
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
+            # Write audio as WAV file
+            sf.write(tmp_file.name, resampled_audio, target_sample_rate)
+            tmp_path = tmp_file.name
+        
+        try:
+            # Transcribe with NeMo
+            result = asr_model.transcribe([tmp_path])
+            
+            # Debug logging to understand result format
+            logger.info(f"NeMo result type: {type(result)}")
+            if result and len(result) > 0:
+                logger.info(f"First result type: {type(result[0])}")
+                logger.info(f"First result content: {result[0]}")
+            
+            if result and len(result) > 0:
+                # Handle different NeMo result formats
+                if hasattr(result[0], 'text'):
+                    # If result has .text attribute (newer NeMo versions)
+                    raw_text = result[0].text
+                    logger.info(f"Using .text attribute: {raw_text}")
+                elif isinstance(result[0], str):
+                    # If result is directly a string
+                    raw_text = result[0]
+                    logger.info(f"Using direct string: {raw_text}")
+                else:
+                    # If result is some other format, convert to string
+                    raw_text = str(result[0])
+                    logger.info(f"Using str() conversion: {raw_text}")
+                
+                # Ensure raw_text is a string before processing
+                if not isinstance(raw_text, str):
+                    raw_text = str(raw_text)
+                
+                # Only process if we have actual text content
+                if raw_text and raw_text.strip():
+                    # Convert Arabic numbers to digits
+
+                    logger.info(f"before sending to FXN--- {raw_text}")
+                    cleaned_text = replace_arabic_numbers(raw_text)
+                    logger.info(f"after FXN--- {cleaned_text}")
+                    end_time = time.time()
+                    
+                    if cleaned_text.strip():
+                        logger.info(f"NeMo transcription: '{cleaned_text}' (processed in {end_time - start_time:.2f}s)")
+                    
+                    return cleaned_text.strip()
+                else:
+                    logger.info("No transcription text found")
+                    return ""
+            else:
+                logger.info("No results from NeMo transcription")
+                return ""
+                
+        finally:
+            # Clean up temporary file
+            if os.path.exists(tmp_path):
+                os.remove(tmp_path)
+        
+    except Exception as e:
+        logger.error(f"Error during NeMo transcription: {e}")
+        return ""
+
+class JambonzSTTHandler:
+    def __init__(self, websocket):
+        self.websocket = websocket
+        self.audio_buffer = None
+        self.config = {}
+        self.running = False
+        self.transcription_task = None
+        
+        # Auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.silence_timeout = 2.0
+        self.min_interim_count = 2
+        self.auto_final_task = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.segment_number = 0
+        self.last_partial = ""
+        
+        # Processing tracking
+        self.processing_count = 0
+        
+    async def start_processing(self, start_message):
+        """Initialize with start message from jambonz"""
+        self.config = {
+            "language": start_message.get("language", "ar-EG"),
+            "format": start_message.get("format", "raw"),
+            "encoding": start_message.get("encoding", "LINEAR16"),
+            "sample_rate": start_message.get("sampleRateHz", 8000),
+            "interim_results": True,  # Always enable for internal processing
+            "options": start_message.get("options", {})
+        }
+        
+        logger.info(f"NeMo STT session started with config: {self.config}")
+        
+        # Initialize audio buffer
+        self.audio_buffer = JambonzAudioBuffer(
+            sample_rate=self.config["sample_rate"],
+            chunk_duration=1.0  # 1 second chunks for NeMo
+        )
+        
+        # Reset session variables
+        self.running = True
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.segment_number = 0
+        self.processing_count = 0
+        self.last_partial = ""
+        
+        # Start background transcription task
+        self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+        # Start auto-final detection task
+        self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+    
+    async def stop_processing(self):
+        """Stop current processing session"""
+        logger.info("Stopping NeMo STT session...")
+        self.running = False
+        
+        # Cancel background tasks
+        for task in [self.transcription_task, self.auto_final_task]:
+            if task:
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+        
+        # Send final transcription if not already sent
+        if not self.final_sent and self.accumulated_transcript.strip():
+            await self.send_transcription(self.accumulated_transcript, is_final=True)
+        
+        # Process any remaining audio for comprehensive final transcription
+        if self.audio_buffer:
+            all_audio = self.audio_buffer.get_all_audio()
+            if len(all_audio) > 0 and not self.final_sent:
+                loop = asyncio.get_event_loop()
+                final_transcription = await loop.run_in_executor(
+                    executor, 
+                    transcribe_with_nemo, 
+                    all_audio,
+                    self.config["sample_rate"]
+                )
+                
+                if final_transcription.strip():
+                    await self.send_transcription(final_transcription, is_final=True)
+        
+        # Clear audio buffer
+        if self.audio_buffer:
+            self.audio_buffer.clear()
+        
+        logger.info("NeMo STT session stopped")
+    
+    async def start_new_segment(self):
+        """Start a new transcription segment"""
+        self.segment_number += 1
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.last_partial = ""
+        self.processing_count = 0
+        
+        if self.audio_buffer:
+            self.audio_buffer.reset_for_new_segment()
+        
+        logger.info(f"Started new transcription segment #{self.segment_number}")
+    
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer"""
+        if self.audio_buffer and self.running:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+    
+    async def _process_audio_chunks(self):
+        """Process audio chunks for interim results"""
+        while self.running:
+            try:
+                if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+                    chunk_signal = self.audio_buffer.get_chunk_for_processing()
+                    if chunk_signal is not None:
+                        all_audio = self.audio_buffer.get_all_audio()
+                        
+                        if len(all_audio) > 0 and self.audio_buffer.is_speech(all_audio[-self.audio_buffer.chunk_samples:]):
+                            loop = asyncio.get_event_loop()
+                            transcription = await loop.run_in_executor(
+                                executor, 
+                                transcribe_with_nemo, 
+                                all_audio,
+                                self.config["sample_rate"]
+                            )
+                            
+                            if transcription.strip():
+                                self.processing_count += 1
+                                self.accumulated_transcript = transcription
+                                
+                                if transcription != self.last_partial or self.interim_count == 0:
+                                    self.last_partial = transcription
+                                    self.interim_count += 1
+                                    self.last_interim_time = time.time()
+                                    logger.info(f"Updated interim_count to {self.interim_count} for transcript: '{transcription}'")
+                                else:
+                                    self.last_interim_time = time.time()
+                
+                await asyncio.sleep(0.1)  # Check every 100ms
+                
+            except Exception as e:
+                logger.error(f"Error in chunk processing: {e}")
+                await asyncio.sleep(0.1)
+    
+    async def _monitor_for_auto_final(self):
+        """Monitor for auto-final conditions"""
+        while self.running:
+            try:
+                current_time = time.time()
+                
+                if (self.interim_count >= self.min_interim_count and 
+                    self.last_interim_time is not None and 
+                    (current_time - self.last_interim_time) >= self.silence_timeout and
+                    not self.final_sent and
+                    self.accumulated_transcript.strip()):
+                    
+                    logger.info(f"Auto-final triggered for segment #{self.segment_number}")
+                    
+                    await self.send_transcription(self.accumulated_transcript, is_final=True)
+                    await self.start_new_segment()
+                
+                await asyncio.sleep(0.5)  # Check every 500ms
+                
+            except Exception as e:
+                logger.error(f"Error in auto-final monitoring: {e}")
+                await asyncio.sleep(0.5)
+    
+    async def send_transcription(self, text, is_final=True, confidence=0.9):
+        """Send transcription in jambonz format"""
+        try:
+            message = {
+                "type": "transcription",
+                "is_final": True,  # Always send as final
+                "alternatives": [
+                    {
+                        "transcript": text,
+                        "confidence": confidence
+                    }
+                ],
+                "language": self.config.get("language", "ar-EG"),
+                "channel": 1
+            }
+            
+            await self.websocket.send(json.dumps(message))
+            self.final_sent = True
+            
+            logger.info(f"Sent FINAL transcription to Jambonz: '{text}'")
+            
+        except Exception as e:
+            logger.error(f"Error sending transcription: {e}")
+    
+    async def send_error(self, error_message):
+        """Send error message in jambonz format"""
+        try:
+            message = {
+                "type": "error",
+                "error": error_message
+            }
+            await self.websocket.send(json.dumps(message))
+            logger.error(f"Sent error: {error_message}")
+        except Exception as e:
+            logger.error(f"Error sending error message: {e}")
+
+async def handle_jambonz_websocket(websocket):
+    """Handle jambonz WebSocket connections"""
+    
+    client_id = f"jambonz_{id(websocket)}"
+    logger.info(f"New NeMo jambonz connection: {client_id}")
+    
+    handler = JambonzSTTHandler(websocket)
+    
+    try:
+        async for message in websocket:
+            try:
+                if isinstance(message, str):
+                    data = json.loads(message)
+                    message_type = data.get("type")
+                    
+                    if message_type == "start":
+                        logger.info(f"Received start message: {data}")
+                        await handler.start_processing(data)
+                        
+                    elif message_type == "stop":
+                        logger.info("Received stop message - closing WebSocket")
+                        await handler.stop_processing()
+                        await websocket.close(code=1000, reason="Session stopped by client")
+                        break
+                        
+                    else:
+                        logger.warning(f"Unknown message type: {message_type}")
+                        await handler.send_error(f"Unknown message type: {message_type}")
+                
+                else:
+                    # Handle binary audio data
+                    if not handler.running or handler.audio_buffer is None:
+                        logger.warning("Received audio data outside of active session")
+                        await handler.send_error("Received audio before start message or after stop")
+                        continue
+                    
+                    await handler.add_audio_data(message)
+            
+            except json.JSONDecodeError as e:
+                logger.error(f"JSON decode error: {e}")
+                await handler.send_error(f"Invalid JSON: {str(e)}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+                await handler.send_error(f"Processing error: {str(e)}")
+    
+    except websockets.exceptions.ConnectionClosed:
+        logger.info(f"NeMo jambonz connection closed: {client_id}")
+    except Exception as e:
+        logger.error(f"NeMo jambonz WebSocket error: {e}")
+        try:
+            await handler.send_error(str(e))
+        except:
+            pass
+    finally:
+        if handler.running:
+            await handler.stop_processing()
+        logger.info(f"NeMo jambonz connection ended: {client_id}")
+
+async def main():
+    """Start the NeMo jambonz STT WebSocket server"""
+    logger.info("Starting NeMo Jambonz STT WebSocket server on port 3007...")
+    
+    # Start WebSocket server
+    server = await websockets.serve(
+        handle_jambonz_websocket,
+        "0.0.0.0", 
+        3007,
+        ping_interval=20,
+        ping_timeout=10,
+        close_timeout=10
+    )
+    
+    logger.info("NeMo Jambonz STT WebSocket server started on ws://0.0.0.0:3007")
+    logger.info("Ready to handle jambonz STT requests with NeMo FastConformer")
+    logger.info("FEATURES:")
+    logger.info("- Arabic ASR using NeMo FastConformer model")
+    logger.info("- Arabic number word to digit conversion")
+    logger.info("- Continuous transcription with segmentation")
+    logger.info("- Voice Activity Detection")
+    
+    # Wait for the server to close
+    await server.wait_closed()
+
+if __name__ == "__main__":
+    print("=" * 80)
+    print("NeMo FastConformer Jambonz STT Server")
+    print("=" * 80)
+    print("Model: NeMo FastConformer Arabic ASR")
+    print("WebSocket Port: 3007")
+    print("Protocol: jambonz STT API")
+    print("Audio Format: LINEAR16 PCM @ 8kHz → 16kHz")
+    print("Language: Arabic with number conversion")
+    print("=" * 80)
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nShutting down NeMo server...")
+    except Exception as e:
+        print(f"Server error: {e}")

aqib-whipser4-arabic.py

@@ -0,0 +1,654 @@
+import torch
+import asyncio
+import websockets
+import json
+import threading
+import numpy as np
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, WhisperTokenizer, pipeline
+import subprocess
+import logging
+import time
+from concurrent.futures import ThreadPoolExecutor
+import struct
+import re
+
+# Arabic number conversion imports
+try:
+    from pyarabic.number import text2number
+    arabic_numbers_available = True
+    print("Arabic number conversion available")
+except ImportError:
+    arabic_numbers_available = False
+    print("pyarabic not available - install with: pip install pyarabic")
+    print("Arabic numbers will not be converted to digits")
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def convert_arabic_numbers_in_sentence(sentence: str) -> str:
+    """
+    Replace Arabic number words in a sentence with digits,
+    preserving all other words and punctuation.
+    Handles common spelling variants and zero explicitly.
+    """
+    try:
+        print("Fxn called--------------")
+
+        # --- Normalization step ---
+        replacements = {
+            "اربعة": "أربعة",
+            "اربع": "أربع",
+            "اثنين": "اثنان",
+            "اتنين": "اثنان",   # Egyptian variant
+            "ثلاث": "ثلاثة",
+            "خمس": "خمسة",
+            "ست": "ستة",
+            "سبع": "سبعة",
+            "ثمان": "ثمانية",
+            "تسع": "تسعة",
+            "عشر": "عشرة",
+        }
+        for wrong, correct in replacements.items():
+            sentence = re.sub(rf"\b{wrong}\b", correct, sentence)
+
+        # --- Split by whitespace but keep spaces ---
+        words = re.split(r'(\s+)', sentence)
+        converted_words = []
+
+        for word in words:
+            stripped = word.strip()
+            if not stripped:  # skip spaces
+                converted_words.append(word)
+                continue
+
+            try:
+                num = text2number(stripped)
+
+                # Accept valid numbers, including zero explicitly
+                if isinstance(num, int):
+                    if num != 0 or stripped == "صفر":
+                        converted_words.append(str(num))
+                    else:
+                        converted_words.append(word)
+                else:
+                    converted_words.append(word)
+
+            except Exception:
+                converted_words.append(word)
+
+        return ''.join(converted_words)
+
+    except Exception as e:
+        logger.warning(f"Error converting Arabic numbers: {e}")
+        return sentence
+
+
+# Try to install flash-attn if not available
+try:
+    import flash_attn
+    use_flash_attn = True
+except ImportError:
+    print("Flash attention not available, using standard attention")
+    use_flash_attn = False
+    try:
+        subprocess.run(
+            "pip install websockets",
+            shell=True,
+            check=False
+        )
+        subprocess.run(
+            "pip install flash-attn --no-build-isolation",
+            shell=True,
+            check=False
+        )
+    except:
+        pass
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+MODEL_NAME = "openai/whisper-large-v3-turbo"
+
+print(f"Using device: {device}")
+print(f"CUDA available: {torch.cuda.is_available()}")
+if torch.cuda.is_available():
+    print(f"GPU: {torch.cuda.get_device_name(0)}")
+
+# Model initialization with fallback for attention implementation
+try:
+    if use_flash_attn and torch.cuda.is_available():
+        model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            MODEL_NAME, 
+            torch_dtype=torch_dtype, 
+            low_cpu_mem_usage=True, 
+            use_safetensors=True, 
+            attn_implementation="flash_attention_2"
+        )
+    else:
+        model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            MODEL_NAME, 
+            torch_dtype=torch_dtype, 
+            low_cpu_mem_usage=True, 
+            use_safetensors=True
+        )
+except Exception as e:
+    print(f"Error loading model with flash attention: {e}")
+    model = AutoModelForSpeechSeq2Seq.from_pretrained(
+        MODEL_NAME, 
+        torch_dtype=torch_dtype, 
+        low_cpu_mem_usage=True, 
+        use_safetensors=True
+    )
+
+model.to(device)
+
+processor = AutoProcessor.from_pretrained(MODEL_NAME)
+tokenizer = WhisperTokenizer.from_pretrained(MODEL_NAME)
+
+# Thread pool for processing audio
+executor = ThreadPoolExecutor(max_workers=4)
+
+class JambonzAudioBuffer:
+    def __init__(self, sample_rate=8000, chunk_duration=1.0):
+        self.sample_rate = sample_rate
+        self.chunk_duration = chunk_duration
+        self.chunk_samples = int(chunk_duration * sample_rate)
+        
+        self.buffer = np.array([], dtype=np.float32)
+        self.lock = threading.Lock()
+        self.total_audio = np.array([], dtype=np.float32)
+        
+        # Voice Activity Detection (simple energy-based)
+        self.silence_threshold = 0.01
+        self.min_speech_samples = int(0.3 * sample_rate)  # 300ms minimum speech
+        
+    def add_audio(self, audio_data):
+        with self.lock:
+            self.buffer = np.concatenate([self.buffer, audio_data])
+            self.total_audio = np.concatenate([self.total_audio, audio_data])
+    
+    def has_chunk_ready(self):
+        with self.lock:
+            return len(self.buffer) >= self.chunk_samples
+    
+    def is_speech(self, audio_chunk):
+        """Simple VAD based on energy"""
+        if len(audio_chunk) < self.min_speech_samples:
+            return False
+        energy = np.mean(np.abs(audio_chunk))
+        return energy > self.silence_threshold
+    
+    def get_chunk_for_processing(self):
+        """Get audio chunk for processing - but don't remove it from buffer for interim results"""
+        with self.lock:
+            if len(self.buffer) < self.chunk_samples:
+                return None
+            
+            # For interim results, we want to trigger processing but keep accumulating audio
+            # So we just return a signal that we have enough audio, but don't consume it
+            return np.array([1])  # Return a dummy array to signal chunk is ready
+    
+    def get_all_audio(self):
+        """Get all accumulated audio for final transcription"""
+        with self.lock:
+            return self.total_audio.copy()
+    
+    def clear(self):
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+
+def linear16_to_audio(audio_bytes, sample_rate=8000):
+    """Convert LINEAR16 PCM bytes to numpy array (jambonz format)"""
+    try:
+        # jambonz sends LINEAR16 PCM at 8kHz
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16)
+        # Convert to float32 and normalize
+        audio_array = audio_array.astype(np.float32) / 32768.0
+        return audio_array
+    except Exception as e:
+        logger.error(f"Error converting LINEAR16 to audio: {e}")
+        return np.array([], dtype=np.float32)
+
+def resample_audio(audio_data, source_rate, target_rate):
+    """Simple resampling from 8kHz to 16kHz for Whisper"""
+    if source_rate == target_rate:
+        return audio_data
+    
+    # Simple linear interpolation resampling
+    ratio = target_rate / source_rate
+    indices = np.arange(0, len(audio_data), 1/ratio)
+    indices = indices[indices < len(audio_data)]
+    resampled = np.interp(indices, np.arange(len(audio_data)), audio_data)
+    
+    # Ensure proper float32 dtype for consistency
+    return resampled.astype(np.float32)
+
+def transcribe_chunk_direct(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio chunk using model's generate method directly"""
+    try:
+        if len(audio_data) == 0:
+            return ""
+        
+        # Resample from 8kHz to 16kHz for Whisper
+        resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        
+        # Ensure minimum length for Whisper
+        min_samples = int(0.1 * target_sample_rate)  # 100ms minimum
+        if len(resampled_audio) < min_samples:
+            return ""
+        
+        start_time = time.time()
+        
+        # Prepare input features with proper dtype
+        input_features = processor(
+            resampled_audio, 
+            sampling_rate=target_sample_rate, 
+            return_tensors="pt"
+        ).input_features
+        
+        # Ensure correct dtype and device
+        input_features = input_features.to(device=device, dtype=torch_dtype)
+        
+        # Create attention mask to avoid warnings
+        attention_mask = torch.ones(
+            input_features.shape[:-1], 
+            dtype=torch.long, 
+            device=device
+        )
+        
+        # Generate transcription using model directly
+        with torch.no_grad():
+            predicted_ids = model.generate(
+                input_features,
+                attention_mask=attention_mask,
+                max_new_tokens=128,
+                do_sample=False,
+                temperature=0.0,
+                num_beams=1,
+                language="ar",
+                task="transcribe",
+                pad_token_id=tokenizer.pad_token_id,
+                eos_token_id=tokenizer.eos_token_id
+            )
+        
+        # Decode the transcription
+        transcription = tokenizer.batch_decode(
+            predicted_ids, 
+            skip_special_tokens=True
+        )[0].strip()
+        
+        end_time = time.time()
+        
+        logger.info(f"Direct transcription completed in {end_time - start_time:.2f}s: '{transcription}'")
+        return transcription
+        
+    except Exception as e:
+        logger.error(f"Error during direct transcription: {e}")
+        return ""
+
+class JambonzSTTHandler:
+    def __init__(self, websocket):
+        self.websocket = websocket
+        self.audio_buffer = None
+        self.config = {}
+        self.running = True
+        self.transcription_task = None
+        self.full_transcript = ""
+        self.last_partial = ""
+        
+        # Auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.silence_timeout = 1.5  # 3 seconds of silence to trigger final
+        self.min_interim_count = 1  # Minimum interim results before considering final
+        self.auto_final_task = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        
+    async def start_processing(self, start_message):
+        """Initialize with start message from jambonz"""
+        self.config = {
+            "language": start_message.get("language", "ar-EG"),
+            "format": start_message.get("format", "raw"),
+            "encoding": start_message.get("encoding", "LINEAR16"),
+            "sample_rate": start_message.get("sampleRateHz", 8000),
+            "interim_results": start_message.get("interimResults", True),
+            "options": start_message.get("options", {})
+        }
+        
+        logger.info(f"STT session started with config: {self.config}")
+        
+        # Initialize audio buffer
+        self.audio_buffer = JambonzAudioBuffer(
+            sample_rate=self.config["sample_rate"],
+            chunk_duration=1.0  # Process every 1 second
+        )
+        
+        # Reset auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        
+        # Start background transcription task
+        self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+        # Start auto-final detection task
+        self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+    
+    async def stop_processing(self):
+        """Stop processing and send final transcription"""
+        self.running = False
+        
+        # Cancel background tasks
+        if self.transcription_task:
+            self.transcription_task.cancel()
+            try:
+                await self.transcription_task
+            except asyncio.CancelledError:
+                pass
+        
+        if self.auto_final_task:
+            self.auto_final_task.cancel()
+            try:
+                await self.auto_final_task
+            except asyncio.CancelledError:
+                pass
+        
+        # Send final transcription if not already sent
+        if not self.final_sent and self.accumulated_transcript.strip():
+            await self.send_transcription(self.accumulated_transcript, is_final=True)
+        
+        # Also process any remaining audio for comprehensive final transcription
+        if self.audio_buffer:
+            all_audio = self.audio_buffer.get_all_audio()
+            if len(all_audio) > 0 and not self.final_sent:
+                loop = asyncio.get_event_loop()
+                final_transcription = await loop.run_in_executor(
+                    executor, 
+                    transcribe_chunk_direct, 
+                    all_audio,
+                    self.config["sample_rate"]
+                )
+                
+                if final_transcription.strip():
+                    # Send comprehensive final transcription
+                    await self.send_transcription(final_transcription, is_final=True)
+        
+        logger.info("STT session ended")
+    
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer"""
+        if self.audio_buffer:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+    
+    async def _process_audio_chunks(self):
+        """Process audio chunks for interim results"""
+        while self.running and self.config.get("interim_results", False):
+            try:
+                if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+                    chunk_signal = self.audio_buffer.get_chunk_for_processing()
+                    if chunk_signal is not None:
+                        # Get all accumulated audio so far for complete transcription
+                        all_audio = self.audio_buffer.get_all_audio()
+                        
+                        # Only process if we have actual speech content
+                        if len(all_audio) > 0 and self.audio_buffer.is_speech(all_audio[-self.audio_buffer.chunk_samples:]):
+                            # Run transcription on all accumulated audio
+                            loop = asyncio.get_event_loop()
+                            transcription = await loop.run_in_executor(
+                                executor, 
+                                transcribe_chunk_direct, 
+                                all_audio,
+                                self.config["sample_rate"]
+                            )
+                            
+                            if transcription.strip() and transcription != self.last_partial:
+                                self.last_partial = transcription
+                                self.accumulated_transcript = transcription  # Update accumulated transcript
+                                self.interim_count += 1
+                                self.last_interim_time = time.time()
+                                
+                                # Send interim result
+                                await self.send_transcription(transcription, is_final=False)
+                                
+                                logger.info(f"Interim #{self.interim_count}: '{transcription}'")
+                
+                # Small delay to prevent excessive processing
+                await asyncio.sleep(0.1)
+                
+            except Exception as e:
+                logger.error(f"Error in chunk processing: {e}")
+                await asyncio.sleep(1)
+    
+    async def _monitor_for_auto_final(self):
+        """Monitor for auto-final conditions: 3 seconds silence after 3+ interim results"""
+        while self.running:
+            try:
+                current_time = time.time()
+                
+                # Check if we should send auto-final transcription
+                if (self.interim_count >= self.min_interim_count and 
+                    self.last_interim_time is not None and 
+                    (current_time - self.last_interim_time) >= self.silence_timeout and
+                    not self.final_sent and
+                    self.accumulated_transcript.strip()):
+                    
+                    logger.info(f"Auto-final triggered: {self.interim_count} interim results, "
+                              f"{current_time - self.last_interim_time:.1f}s silence")
+                    
+                    # Send the accumulated transcript as final
+                    await self.send_transcription(self.accumulated_transcript, is_final=True)
+                    self.final_sent = True
+                    
+                    # Reset counters for potential next utterance
+                    self.interim_count = 0
+                    self.last_interim_time = None
+                    self.accumulated_transcript = ""
+                
+                # Check every 0.5 seconds
+                await asyncio.sleep(0.5)
+                
+            except Exception as e:
+                logger.error(f"Error in auto-final monitoring: {e}")
+                await asyncio.sleep(1)
+    
+    # async def send_transcription(self, text, is_final=False, confidence=0.9):
+    #     """Send transcription in jambonz format with Arabic number conversion"""
+    #     try:
+    #         # Convert Arabic numbers to digits before sending
+    #         original_text = text
+    #         converted_text = convert_arabic_numbers_in_sentence(text)
+            
+    #         # Log the conversion if numbers were found and converted
+    #         if original_text != converted_text:
+    #             logger.info(f"Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+            
+    #         message = {
+    #             "type": "transcription",
+    #             "is_final": is_final,
+    #             "alternatives": [
+    #                 {
+    #                     "transcript": converted_text,
+    #                     "confidence": confidence
+    #                 }
+    #             ],
+    #             "language": self.config.get("language", "ar-EG"),
+    #             "channel": 1
+    #         }
+            
+    #         await self.websocket.send(json.dumps(message))
+    #         logger.info(f"Sent {'FINAL' if is_final else 'interim'} transcription: '{converted_text}'")
+            
+    #         if is_final:
+    #             self.final_sent = True
+            
+    #     except Exception as e:
+    #         logger.error(f"Error sending transcription: {e}")
+    
+
+
+    async def send_transcription(self, text, is_final=False, confidence=0.9):
+        """Send transcription in jambonz format with Arabic number conversion, only for final results"""
+        try:
+            if not is_final:
+                # Do nothing for interim results
+                logger.debug("Skipping interim transcription (not final).")
+                return
+
+            # Convert Arabic numbers only for final transcripts
+            original_text = text
+            converted_text = convert_arabic_numbers_in_sentence(text)
+
+            # Log the conversion if numbers were found and converted
+            if original_text != converted_text:
+                logger.info(f"Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+
+            message = {
+                "type": "transcription",
+                "is_final": True,
+                "alternatives": [
+                    {
+                        "transcript": converted_text,
+                        "confidence": confidence
+                    }
+                ],
+                "language": self.config.get("language", "ar-EG"),
+                "channel": 1
+            }
+
+            # Send only final messages
+            await self.websocket.send(json.dumps(message))
+            logger.info(f"Sent FINAL transcription: '{converted_text}'")
+
+            self.final_sent = True
+
+        except Exception as e:
+            logger.error(f"Error sending transcription: {e}")
+
+
+
+
+    async def send_error(self, error_message):
+        """Send error message in jambonz format"""
+        try:
+            message = {
+                "type": "error",
+                "error": error_message
+            }
+            await self.websocket.send(json.dumps(message))
+            logger.error(f"Sent error: {error_message}")
+        except Exception as e:
+            logger.error(f"Error sending error message: {e}")
+
+async def handle_jambonz_websocket(websocket):
+    """Handle jambonz WebSocket connections"""
+    
+    client_id = f"jambonz_{id(websocket)}"
+    logger.info(f"New jambonz connection: {client_id}")
+    
+    handler = JambonzSTTHandler(websocket)
+    
+    try:
+        async for message in websocket:
+            try:
+                if isinstance(message, str):
+                    # Handle JSON control messages
+                    data = json.loads(message)
+                    message_type = data.get("type")
+                    
+                    if message_type == "start":
+                        logger.info(f"Received start message: {data}")
+                        await handler.start_processing(data)
+                        
+                    elif message_type == "stop":
+                        logger.info("Received stop message")
+                        await handler.stop_processing()
+                        # Close websocket after final transcription
+                        await websocket.close(code=1000, reason="Session completed")
+                        break
+                        
+                    else:
+                        logger.warning(f"Unknown message type: {message_type}")
+                        await handler.send_error(f"Unknown message type: {message_type}")
+                
+                else:
+                    # Handle binary audio data (LINEAR16 PCM)
+                    if handler.audio_buffer is None:
+                        await handler.send_error("Received audio before start message")
+                        continue
+                    
+                    await handler.add_audio_data(message)
+            
+            except json.JSONDecodeError as e:
+                logger.error(f"JSON decode error: {e}")
+                await handler.send_error(f"Invalid JSON: {str(e)}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+                await handler.send_error(f"Processing error: {str(e)}")
+    
+    except websockets.exceptions.ConnectionClosed:
+        logger.info(f"jambonz connection closed: {client_id}")
+    except Exception as e:
+        logger.error(f"jambonz WebSocket error: {e}")
+        try:
+            await handler.send_error(str(e))
+        except:
+            pass
+    finally:
+        if handler.running:
+            await handler.stop_processing()
+        logger.info(f"jambonz connection ended: {client_id}")
+
+async def main():
+    """Start the jambonz STT WebSocket server"""
+    logger.info("Starting Jambonz Custom STT WebSocket server on port 3006...")
+    
+    # Start WebSocket server
+    server = await websockets.serve(
+        handle_jambonz_websocket,
+        "0.0.0.0", 
+        3006,
+        ping_interval=20,
+        ping_timeout=10,
+        close_timeout=10
+    )
+    
+    logger.info("Jambonz Custom STT WebSocket server started on ws://0.0.0.0:3006")
+    logger.info("Ready to handle jambonz STT requests")
+    logger.info("- Expects LINEAR16 PCM audio at 8kHz")
+    logger.info("- Supports interim results with auto-final detection")
+    logger.info("- Auto-final: 3+ interim results + 1.3s silence")
+    logger.info("- Resamples to 16kHz for Whisper processing")
+    logger.info("- Converts Arabic numbers to digits before sending")
+    
+    # Wait for the server to close
+    await server.wait_closed()
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print("Jambonz Custom STT Server with Whisper + Arabic Numbers")
+    print("=" * 60)
+    print(f"Model: {MODEL_NAME}")
+    print(f"Device: {device}")
+    print("WebSocket Port: 3006")
+    print("Protocol: jambonz STT API")
+    print("Audio Format: LINEAR16 PCM @ 8kHz")
+    print("Auto-Final: 2+ speech activities + 1.3s silence")
+    print("Arabic Numbers: Converted to digits in FINAL transcriptions only")
+    print("Interim Results: DISABLED (final transcription only)")
+    if arabic_numbers_available:
+        print("✓ pyarabic library available for number conversion")
+    else:
+        print("✗ pyarabic library not available - install with: pip install pyarabic")
+    print("=" * 60)
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nShutting down server...")
+    except Exception as e:
+        print(f"Server error: {e}")

aqib-whipser_ft-arabic_denoiser_meta.py

@@ -0,0 +1,787 @@
+# import torch
+# import asyncio
+# import websockets
+# import json
+# import threading
+# import numpy as np
+# from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, WhisperTokenizer, pipeline , WhisperForConditionalGeneration, WhisperProcessor
+# import subprocess
+# import logging
+# import time
+# from concurrent.futures import ThreadPoolExecutor
+# import struct
+# import re
+# 3 - 10 - 2025
+import torch
+import asyncio
+import websockets
+import json
+import threading
+import numpy as np
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, WhisperTokenizer
+import subprocess
+import logging
+import time
+from concurrent.futures import ThreadPoolExecutor
+import re
+
+# --- Denoiser added ---
+try:
+    import noisereduce as nr
+    denoiser_available = True
+    print("Denoiser available (using noisereduce)")
+except ImportError:
+    denoiser_available = False
+    print("noisereduce not available - install with: pip install noisereduce")
+##############################################################################################
+# Arabic number conversion imports
+try:
+    from pyarabic.number import text2number
+    arabic_numbers_available = True
+    print("Arabic number conversion available")
+except ImportError:
+    arabic_numbers_available = False
+    print("pyarabic not available - install with: pip install pyarabic")
+    print("Arabic numbers will not be converted to digits")
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+# 3 - 10 - 2025
+# def denoise_audio(audio_data, sample_rate=16000):
+#     """Apply noise reduction to audio using noisereduce."""
+#     if not denoiser_available or len(audio_data) == 0:
+#         return audio_data
+#     try:
+#         reduced = nr.reduce_noise(y=audio_data, sr=sample_rate)
+#         return reduced.astype(np.float32)
+#     except Exception as e:
+#         logger.warning(f"Denoiser failed: {e}")
+#         return audio_data
+#############################################################################################
+def convert_arabic_numbers_in_sentence(sentence: str) -> str:
+    """
+    Replace Arabic number words in a sentence with digits,
+    preserving all other words and punctuation.
+    Handles common spelling variants and zero explicitly.
+    """
+    try:
+        print("Fxn called--------------")
+
+        # --- Normalization step ---
+        replacements = {
+            "اربعة": "أربعة",
+            "اربع": "أربع",
+            "اثنين": "اثنان",
+            "اتنين": "اثنان",   # Egyptian variant
+            "ثلاث": "ثلاثة",
+            "خمس": "خمسة",
+            "ست": "ستة",
+            "سبع": "سبعة",
+            "ثمان": "ثمانية",
+            "تسع": "تسعة",
+            "عشر": "عشرة",
+        }
+        for wrong, correct in replacements.items():
+            sentence = re.sub(rf"\b{wrong}\b", correct, sentence)
+
+        # --- Split by whitespace but keep spaces ---
+        words = re.split(r'(\s+)', sentence)
+        converted_words = []
+
+        for word in words:
+            stripped = word.strip()
+            if not stripped:  # skip spaces
+                converted_words.append(word)
+                continue
+
+            try:
+                num = text2number(stripped)
+
+                # Accept valid numbers, including zero explicitly
+                if isinstance(num, int):
+                    if num != 0 or stripped == "صفر":
+                        converted_words.append(str(num))
+                    else:
+                        converted_words.append(word)
+                else:
+                    converted_words.append(word)
+
+            except Exception:
+                converted_words.append(word)
+
+        return ''.join(converted_words)
+
+    except Exception as e:
+        logger.warning(f"Error converting Arabic numbers: {e}")
+        return sentence
+
+
+# Try to install flash-attn if not available
+try:
+    import flash_attn
+    use_flash_attn = True
+except ImportError:
+    print("Flash attention not available, using standard attention")
+    use_flash_attn = False
+    try:
+        subprocess.run(
+            "pip install websockets",
+            shell=True,
+            check=False
+        )
+        subprocess.run(
+            "pip install flash-attn --no-build-isolation",
+            shell=True,
+            check=False
+        )
+    except:
+        pass
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+# --- Facebook Denoiser added ---
+try:
+    import torchaudio
+    from denoiser import pretrained
+    # Load DNS64 pretrained model (auto-downloads if not cached)
+    denoiser_model = pretrained.dns64().to(device)
+    denoiser_model.eval()
+    denoiser_available = True
+    print("facebook/denoiser loaded successfully")
+except ImportError as e:
+    denoiser_available = False
+    print("facebook/denoiser not available - install with: pip install denoiser torchaudio")
+    denoiser_model = None
+
+
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+MODEL_NAME = "alaatiger989/FT_Arabic_Whisper_V1_1"#"openai/whisper-large-v3-turbo"
+
+print(f"Using device: {device}")
+print(f"CUDA available: {torch.cuda.is_available()}")
+if torch.cuda.is_available():
+    print(f"GPU: {torch.cuda.get_device_name(0)}")
+
+# Model initialization with fallback for attention implementation
+try:
+    if use_flash_attn and torch.cuda.is_available():
+        model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            MODEL_NAME, 
+            torch_dtype=torch_dtype, 
+            low_cpu_mem_usage=True, 
+            use_safetensors=True, 
+            attn_implementation="flash_attention_2"
+        )
+    else:
+        model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            MODEL_NAME, 
+            torch_dtype=torch_dtype, 
+            low_cpu_mem_usage=True, 
+            use_safetensors=True
+        )
+except Exception as e:
+    print(f"Error loading model with flash attention: {e}")
+    model = AutoModelForSpeechSeq2Seq.from_pretrained(
+        MODEL_NAME, 
+        torch_dtype=torch_dtype, 
+        low_cpu_mem_usage=True, 
+        use_safetensors=True
+    )
+
+model.to(device)
+
+processor = AutoProcessor.from_pretrained(MODEL_NAME)
+tokenizer = WhisperTokenizer.from_pretrained(MODEL_NAME)
+def denoise_audio(audio_data, sample_rate=16000):
+    """Apply denoising using facebook/denoiser pretrained model."""
+    if denoiser_model is None or len(audio_data) == 0:
+        return audio_data
+    try:
+        audio_tensor = torch.tensor(audio_data, dtype=torch.float32, device=device).unsqueeze(0)
+        with torch.no_grad():
+            denoised_tensor = denoiser_model(audio_tensor, sample_rate=sample_rate)[0]
+        return denoised_tensor.squeeze().cpu().numpy().astype("float32")
+    except Exception as e:
+        print(f"[WARN] Denoiser failed: {e}")
+        return audio_data
+# def denoise_audio(audio_data, sample_rate=16000):
+#     """Apply denoising using facebook/denoiser pretrained model."""
+#     if not denoiser_available or denoiser_model is None or len(audio_data) == 0:
+#         return audio_data
+#     try:
+#         # Convert numpy -> torch tensor
+#         audio_tensor = torch.tensor(audio_data, dtype=torch.float32, device=device).unsqueeze(0)
+#         with torch.no_grad():
+#             denoised_tensor = denoiser_model(audio_tensor)[0]
+#         # Back to numpy
+#         denoised_audio = denoised_tensor.squeeze().cpu().numpy().astype(np.float32)
+#         return denoised_audio
+#     except Exception as e:
+#         logger.warning(f"Denoiser failed: {e}")
+#         return audio_data
+# Thread pool for processing audio
+executor = ThreadPoolExecutor(max_workers=4)
+
+class JambonzAudioBuffer:
+    def __init__(self, sample_rate=8000, chunk_duration=1.0):
+        self.sample_rate = sample_rate
+        self.chunk_duration = chunk_duration
+        self.chunk_samples = int(chunk_duration * sample_rate)
+        
+        self.buffer = np.array([], dtype=np.float32)
+        self.lock = threading.Lock()
+        self.total_audio = np.array([], dtype=np.float32)
+        
+        # Voice Activity Detection (simple energy-based)
+        self.silence_threshold = 0.01
+        self.min_speech_samples = int(0.3 * sample_rate)  # 300ms minimum speech
+        
+    def add_audio(self, audio_data):
+        with self.lock:
+            self.buffer = np.concatenate([self.buffer, audio_data])
+            self.total_audio = np.concatenate([self.total_audio, audio_data])
+    
+    def has_chunk_ready(self):
+        with self.lock:
+            return len(self.buffer) >= self.chunk_samples
+    
+    def is_speech(self, audio_chunk):
+        """Simple VAD based on energy"""
+        if len(audio_chunk) < self.min_speech_samples:
+            return False
+        energy = np.mean(np.abs(audio_chunk))
+        return energy > self.silence_threshold
+    
+    def get_chunk_for_processing(self):
+        """Get audio chunk for processing - but don't remove it from buffer for interim results"""
+        with self.lock:
+            if len(self.buffer) < self.chunk_samples:
+                return None
+            
+            # For interim results, we want to trigger processing but keep accumulating audio
+            # So we just return a signal that we have enough audio, but don't consume it
+            return np.array([1])  # Return a dummy array to signal chunk is ready
+    
+    def get_all_audio(self):
+        """Get all accumulated audio for final transcription"""
+        with self.lock:
+            return self.total_audio.copy()
+    
+    def clear(self):
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+
+def linear16_to_audio(audio_bytes, sample_rate=8000):
+    """Convert LINEAR16 PCM bytes to numpy array (jambonz format)"""
+    try:
+        # jambonz sends LINEAR16 PCM at 8kHz
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16)
+        # Convert to float32 and normalize
+        audio_array = audio_array.astype(np.float32) / 32768.0
+        return audio_array
+    except Exception as e:
+        logger.error(f"Error converting LINEAR16 to audio: {e}")
+        return np.array([], dtype=np.float32)
+
+def resample_audio(audio_data, source_rate, target_rate):
+    """Simple resampling from 8kHz to 16kHz for Whisper"""
+    if source_rate == target_rate:
+        return audio_data
+    
+    # Simple linear interpolation resampling
+    ratio = target_rate / source_rate
+    indices = np.arange(0, len(audio_data), 1/ratio)
+    indices = indices[indices < len(audio_data)]
+    resampled = np.interp(indices, np.arange(len(audio_data)), audio_data)
+    
+    # Ensure proper float32 dtype for consistency
+    return resampled.astype(np.float32)
+def transcribe_chunk_direct(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio chunk using model's generate method directly"""
+    try:
+        if len(audio_data) == 0:
+            return ""
+
+        # Resample from 8kHz to 16kHz for Whisper
+        resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+
+        # --- Denoiser added ---
+        resampled_audio = denoise_audio(resampled_audio, sample_rate=target_sample_rate)
+
+        # Ensure minimum length for Whisper
+        min_samples = int(0.1 * target_sample_rate)  # 100ms minimum
+        if len(resampled_audio) < min_samples:
+            return ""
+
+        start_time = time.time()
+
+        # Prepare input features
+        input_features = processor(
+            resampled_audio,
+            sampling_rate=target_sample_rate,
+            return_tensors="pt"
+        ).input_features
+
+        input_features = input_features.to(device=device, dtype=torch_dtype)
+
+        attention_mask = torch.ones(
+            input_features.shape[:-1],
+            dtype=torch.long,
+            device=device
+        )
+
+        with torch.no_grad():
+            predicted_ids = model.generate(
+                input_features,
+                attention_mask=attention_mask,
+                max_new_tokens=128,
+                do_sample=False,
+                temperature=0.0,
+                num_beams=1,
+                language="ar",
+                task="transcribe",
+                pad_token_id=tokenizer.pad_token_id,
+                eos_token_id=tokenizer.eos_token_id
+            )
+
+        transcription = tokenizer.batch_decode(
+            predicted_ids,
+            skip_special_tokens=True
+        )[0].strip()
+
+        end_time = time.time()
+
+        logger.info(f"Direct transcription completed in {end_time - start_time:.2f}s: '{transcription}'")
+        return transcription
+
+    except Exception as e:
+        logger.error(f"Error during direct transcription: {e}")
+        return ""
+# def transcribe_chunk_direct(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+#     """Transcribe audio chunk using model's generate method directly"""
+#     try:
+#         if len(audio_data) == 0:
+#             return ""
+        
+#         # Resample from 8kHz to 16kHz for Whisper
+#         resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        
+#         # Ensure minimum length for Whisper
+#         min_samples = int(0.1 * target_sample_rate)  # 100ms minimum
+#         if len(resampled_audio) < min_samples:
+#             return ""
+        
+#         start_time = time.time()
+        
+#         # Prepare input features with proper dtype
+#         input_features = processor(
+#             resampled_audio, 
+#             sampling_rate=target_sample_rate, 
+#             return_tensors="pt"
+#         ).input_features
+        
+#         # Ensure correct dtype and device
+#         input_features = input_features.to(device=device, dtype=torch_dtype)
+        
+#         # Create attention mask to avoid warnings
+#         attention_mask = torch.ones(
+#             input_features.shape[:-1], 
+#             dtype=torch.long, 
+#             device=device
+#         )
+        
+#         # Generate transcription using model directly
+#         with torch.no_grad():
+#             predicted_ids = model.generate(
+#                 input_features,
+#                 attention_mask=attention_mask,
+#                 max_new_tokens=128,
+#                 do_sample=False,
+#                 temperature=0.0,
+#                 num_beams=1,
+#                 language="ar",
+#                 task="transcribe",
+#                 pad_token_id=tokenizer.pad_token_id,
+#                 eos_token_id=tokenizer.eos_token_id
+#             )
+        
+#         # Decode the transcription
+#         transcription = tokenizer.batch_decode(
+#             predicted_ids, 
+#             skip_special_tokens=True
+#         )[0].strip()
+        
+#         end_time = time.time()
+        
+#         logger.info(f"Direct transcription completed in {end_time - start_time:.2f}s: '{transcription}'")
+#         return transcription
+        
+#     except Exception as e:
+#         logger.error(f"Error during direct transcription: {e}")
+#         return ""
+
+class JambonzSTTHandler:
+    def __init__(self, websocket):
+        self.websocket = websocket
+        self.audio_buffer = None
+        self.config = {}
+        self.running = True
+        self.transcription_task = None
+        self.full_transcript = ""
+        self.last_partial = ""
+        
+        # Auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.silence_timeout = 1.5  # 3 seconds of silence to trigger final
+        self.min_interim_count = 1  # Minimum interim results before considering final
+        self.auto_final_task = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        
+    async def start_processing(self, start_message):
+        """Initialize with start message from jambonz"""
+        self.config = {
+            "language": start_message.get("language", "ar-EG"),
+            "format": start_message.get("format", "raw"),
+            "encoding": start_message.get("encoding", "LINEAR16"),
+            "sample_rate": start_message.get("sampleRateHz", 8000),
+            "interim_results": start_message.get("interimResults", True),
+            "options": start_message.get("options", {})
+        }
+        
+        logger.info(f"STT session started with config: {self.config}")
+        
+        # Initialize audio buffer
+        self.audio_buffer = JambonzAudioBuffer(
+            sample_rate=self.config["sample_rate"],
+            chunk_duration=1.0  # Process every 1 second
+        )
+        
+        # Reset auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        
+        # Start background transcription task
+        self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+        # Start auto-final detection task
+        self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+    
+    async def stop_processing(self):
+        """Stop processing and send final transcription"""
+        self.running = False
+        
+        # Cancel background tasks
+        if self.transcription_task:
+            self.transcription_task.cancel()
+            try:
+                await self.transcription_task
+            except asyncio.CancelledError:
+                pass
+        
+        if self.auto_final_task:
+            self.auto_final_task.cancel()
+            try:
+                await self.auto_final_task
+            except asyncio.CancelledError:
+                pass
+        
+        # Send final transcription if not already sent
+        if not self.final_sent and self.accumulated_transcript.strip():
+            await self.send_transcription(self.accumulated_transcript, is_final=True)
+        
+        # Also process any remaining audio for comprehensive final transcription
+        if self.audio_buffer:
+            all_audio = self.audio_buffer.get_all_audio()
+            if len(all_audio) > 0 and not self.final_sent:
+                loop = asyncio.get_event_loop()
+                final_transcription = await loop.run_in_executor(
+                    executor, 
+                    transcribe_chunk_direct, 
+                    all_audio,
+                    self.config["sample_rate"]
+                )
+                
+                if final_transcription.strip():
+                    # Send comprehensive final transcription
+                    await self.send_transcription(final_transcription, is_final=True)
+        
+        logger.info("STT session ended")
+    
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer"""
+        if self.audio_buffer:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+    
+    async def _process_audio_chunks(self):
+        """Process audio chunks for interim results"""
+        while self.running and self.config.get("interim_results", False):
+            try:
+                if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+                    chunk_signal = self.audio_buffer.get_chunk_for_processing()
+                    if chunk_signal is not None:
+                        # Get all accumulated audio so far for complete transcription
+                        all_audio = self.audio_buffer.get_all_audio()
+                        
+                        # Only process if we have actual speech content
+                        if len(all_audio) > 0 and self.audio_buffer.is_speech(all_audio[-self.audio_buffer.chunk_samples:]):
+                            # Run transcription on all accumulated audio
+                            loop = asyncio.get_event_loop()
+                            transcription = await loop.run_in_executor(
+                                executor, 
+                                transcribe_chunk_direct, 
+                                all_audio,
+                                self.config["sample_rate"]
+                            )
+                            
+                            if transcription.strip() and transcription != self.last_partial:
+                                self.last_partial = transcription
+                                self.accumulated_transcript = transcription  # Update accumulated transcript
+                                self.interim_count += 1
+                                self.last_interim_time = time.time()
+                                
+                                # Send interim result
+                                await self.send_transcription(transcription, is_final=False)
+                                
+                                logger.info(f"Interim #{self.interim_count}: '{transcription}'")
+                
+                # Small delay to prevent excessive processing
+                await asyncio.sleep(0.1)
+                
+            except Exception as e:
+                logger.error(f"Error in chunk processing: {e}")
+                await asyncio.sleep(1)
+    
+    async def _monitor_for_auto_final(self):
+        """Monitor for auto-final conditions: 3 seconds silence after 3+ interim results"""
+        while self.running:
+            try:
+                current_time = time.time()
+                
+                # Check if we should send auto-final transcription
+                if (self.interim_count >= self.min_interim_count and 
+                    self.last_interim_time is not None and 
+                    (current_time - self.last_interim_time) >= self.silence_timeout and
+                    not self.final_sent and
+                    self.accumulated_transcript.strip()):
+                    
+                    logger.info(f"Auto-final triggered: {self.interim_count} interim results, "
+                              f"{current_time - self.last_interim_time:.1f}s silence")
+                    
+                    # Send the accumulated transcript as final
+                    await self.send_transcription(self.accumulated_transcript, is_final=True)
+                    self.final_sent = True
+                    
+                    # Reset counters for potential next utterance
+                    self.interim_count = 0
+                    self.last_interim_time = None
+                    self.accumulated_transcript = ""
+                
+                # Check every 0.5 seconds
+                await asyncio.sleep(0.5)
+                
+            except Exception as e:
+                logger.error(f"Error in auto-final monitoring: {e}")
+                await asyncio.sleep(1)
+    
+    # async def send_transcription(self, text, is_final=False, confidence=0.9):
+    #     """Send transcription in jambonz format with Arabic number conversion"""
+    #     try:
+    #         # Convert Arabic numbers to digits before sending
+    #         original_text = text
+    #         converted_text = convert_arabic_numbers_in_sentence(text)
+            
+    #         # Log the conversion if numbers were found and converted
+    #         if original_text != converted_text:
+    #             logger.info(f"Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+            
+    #         message = {
+    #             "type": "transcription",
+    #             "is_final": is_final,
+    #             "alternatives": [
+    #                 {
+    #                     "transcript": converted_text,
+    #                     "confidence": confidence
+    #                 }
+    #             ],
+    #             "language": self.config.get("language", "ar-EG"),
+    #             "channel": 1
+    #         }
+            
+    #         await self.websocket.send(json.dumps(message))
+    #         logger.info(f"Sent {'FINAL' if is_final else 'interim'} transcription: '{converted_text}'")
+            
+    #         if is_final:
+    #             self.final_sent = True
+            
+    #     except Exception as e:
+    #         logger.error(f"Error sending transcription: {e}")
+    
+
+
+    async def send_transcription(self, text, is_final=False, confidence=0.9):
+        """Send transcription in jambonz format with Arabic number conversion, only for final results"""
+        try:
+            if not is_final:
+                # Do nothing for interim results
+                logger.debug("Skipping interim transcription (not final).")
+                return
+
+            # Convert Arabic numbers only for final transcripts
+            original_text = text
+            converted_text = convert_arabic_numbers_in_sentence(text)
+
+            # Log the conversion if numbers were found and converted
+            if original_text != converted_text:
+                logger.info(f"Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+
+            message = {
+                "type": "transcription",
+                "is_final": True,
+                "alternatives": [
+                    {
+                        "transcript": original_text,#converted_text,
+                        "confidence": confidence
+                    }
+                ],
+                "language": self.config.get("language", "ar-EG"),
+                "channel": 1
+            }
+
+            # Send only final messages
+            await self.websocket.send(json.dumps(message))
+            logger.info(f"Sent FINAL transcription: '{converted_text}'")
+
+            self.final_sent = True
+
+        except Exception as e:
+            logger.error(f"Error sending transcription: {e}")
+
+
+
+
+    async def send_error(self, error_message):
+        """Send error message in jambonz format"""
+        try:
+            message = {
+                "type": "error",
+                "error": error_message
+            }
+            await self.websocket.send(json.dumps(message))
+            logger.error(f"Sent error: {error_message}")
+        except Exception as e:
+            logger.error(f"Error sending error message: {e}")
+
+async def handle_jambonz_websocket(websocket):
+    """Handle jambonz WebSocket connections"""
+    
+    client_id = f"jambonz_{id(websocket)}"
+    logger.info(f"New jambonz connection: {client_id}")
+    
+    handler = JambonzSTTHandler(websocket)
+    
+    try:
+        async for message in websocket:
+            try:
+                if isinstance(message, str):
+                    # Handle JSON control messages
+                    data = json.loads(message)
+                    message_type = data.get("type")
+                    
+                    if message_type == "start":
+                        logger.info(f"Received start message: {data}")
+                        await handler.start_processing(data)
+                        
+                    elif message_type == "stop":
+                        logger.info("Received stop message")
+                        await handler.stop_processing()
+                        # Close websocket after final transcription
+                        await websocket.close(code=1000, reason="Session completed")
+                        break
+                        
+                    else:
+                        logger.warning(f"Unknown message type: {message_type}")
+                        await handler.send_error(f"Unknown message type: {message_type}")
+                
+                else:
+                    # Handle binary audio data (LINEAR16 PCM)
+                    if handler.audio_buffer is None:
+                        await handler.send_error("Received audio before start message")
+                        continue
+                    
+                    await handler.add_audio_data(message)
+            
+            except json.JSONDecodeError as e:
+                logger.error(f"JSON decode error: {e}")
+                await handler.send_error(f"Invalid JSON: {str(e)}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+                await handler.send_error(f"Processing error: {str(e)}")
+    
+    except websockets.exceptions.ConnectionClosed:
+        logger.info(f"jambonz connection closed: {client_id}")
+    except Exception as e:
+        logger.error(f"jambonz WebSocket error: {e}")
+        try:
+            await handler.send_error(str(e))
+        except:
+            pass
+    finally:
+        if handler.running:
+            await handler.stop_processing()
+        logger.info(f"jambonz connection ended: {client_id}")
+
+async def main():
+    """Start the jambonz STT WebSocket server"""
+    logger.info("Starting Jambonz Custom STT WebSocket server on port 3006...")
+    
+    # Start WebSocket server
+    server = await websockets.serve(
+        handle_jambonz_websocket,
+        "0.0.0.0", 
+        3006,
+        ping_interval=20,
+        ping_timeout=10,
+        close_timeout=10
+    )
+    
+    logger.info("Jambonz Custom STT WebSocket server started on ws://0.0.0.0:3006")
+    logger.info("Ready to handle jambonz STT requests")
+    logger.info("- Expects LINEAR16 PCM audio at 8kHz")
+    logger.info("- Supports interim results with auto-final detection")
+    logger.info("- Auto-final: 3+ interim results + 1.3s silence")
+    logger.info("- Resamples to 16kHz for Whisper processing")
+    logger.info("- Converts Arabic numbers to digits before sending")
+    
+    # Wait for the server to close
+    await server.wait_closed()
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print("Jambonz Custom STT Server with Whisper + Arabic Numbers")
+    print("=" * 60)
+    print(f"Model: {MODEL_NAME}")
+    print(f"Device: {device}")
+    print("WebSocket Port: 3006")
+    print("Protocol: jambonz STT API")
+    print("Audio Format: LINEAR16 PCM @ 8kHz")
+    print("Auto-Final: 2+ speech activities + 1.3s silence")
+    print("Arabic Numbers: Converted to digits in FINAL transcriptions only")
+    print("Interim Results: DISABLED (final transcription only)")
+    if arabic_numbers_available:
+        print("✓ pyarabic library available for number conversion")
+    else:
+        print("✗ pyarabic library not available - install with: pip install pyarabic")
+    print("=" * 60)
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nShutting down server...")
+    except Exception as e:
+        print(f"Server error: {e}")

aqib-whipser_ft-arabic_noise_reducer.py

@@ -0,0 +1,746 @@
+# import torch
+# import asyncio
+# import websockets
+# import json
+# import threading
+# import numpy as np
+# from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, WhisperTokenizer, pipeline , WhisperForConditionalGeneration, WhisperProcessor
+# import subprocess
+# import logging
+# import time
+# from concurrent.futures import ThreadPoolExecutor
+# import struct
+# import re
+# 3 - 10 - 2025
+import torch
+import asyncio
+import websockets
+import json
+import threading
+import numpy as np
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, WhisperTokenizer
+import subprocess
+import logging
+import time
+from concurrent.futures import ThreadPoolExecutor
+import re
+
+# --- Denoiser added ---
+try:
+    import noisereduce as nr
+    denoiser_available = True
+    print("Denoiser available (using noisereduce)")
+except ImportError:
+    denoiser_available = False
+    print("noisereduce not available - install with: pip install noisereduce")
+##############################################################################################
+# Arabic number conversion imports
+try:
+    from pyarabic.number import text2number
+    arabic_numbers_available = True
+    print("Arabic number conversion available")
+except ImportError:
+    arabic_numbers_available = False
+    print("pyarabic not available - install with: pip install pyarabic")
+    print("Arabic numbers will not be converted to digits")
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+# 3 - 10 - 2025
+def denoise_audio(audio_data, sample_rate=16000):
+    """Apply noise reduction to audio using noisereduce."""
+    if not denoiser_available or len(audio_data) == 0:
+        return audio_data
+    try:
+        reduced = nr.reduce_noise(y=audio_data, sr=sample_rate)
+        return reduced.astype(np.float32)
+    except Exception as e:
+        logger.warning(f"Denoiser failed: {e}")
+        return audio_data
+#############################################################################################
+def convert_arabic_numbers_in_sentence(sentence: str) -> str:
+    """
+    Replace Arabic number words in a sentence with digits,
+    preserving all other words and punctuation.
+    Handles common spelling variants and zero explicitly.
+    """
+    try:
+        print("Fxn called--------------")
+
+        # --- Normalization step ---
+        replacements = {
+            "اربعة": "أربعة",
+            "اربع": "أربع",
+            "اثنين": "اثنان",
+            "اتنين": "اثنان",   # Egyptian variant
+            "ثلاث": "ثلاثة",
+            "خمس": "خمسة",
+            "ست": "ستة",
+            "سبع": "سبعة",
+            "ثمان": "ثمانية",
+            "تسع": "تسعة",
+            "عشر": "عشرة",
+        }
+        for wrong, correct in replacements.items():
+            sentence = re.sub(rf"\b{wrong}\b", correct, sentence)
+
+        # --- Split by whitespace but keep spaces ---
+        words = re.split(r'(\s+)', sentence)
+        converted_words = []
+
+        for word in words:
+            stripped = word.strip()
+            if not stripped:  # skip spaces
+                converted_words.append(word)
+                continue
+
+            try:
+                num = text2number(stripped)
+
+                # Accept valid numbers, including zero explicitly
+                if isinstance(num, int):
+                    if num != 0 or stripped == "صفر":
+                        converted_words.append(str(num))
+                    else:
+                        converted_words.append(word)
+                else:
+                    converted_words.append(word)
+
+            except Exception:
+                converted_words.append(word)
+
+        return ''.join(converted_words)
+
+    except Exception as e:
+        logger.warning(f"Error converting Arabic numbers: {e}")
+        return sentence
+
+
+# Try to install flash-attn if not available
+try:
+    import flash_attn
+    use_flash_attn = True
+except ImportError:
+    print("Flash attention not available, using standard attention")
+    use_flash_attn = False
+    try:
+        subprocess.run(
+            "pip install websockets",
+            shell=True,
+            check=False
+        )
+        subprocess.run(
+            "pip install flash-attn --no-build-isolation",
+            shell=True,
+            check=False
+        )
+    except:
+        pass
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+MODEL_NAME = "alaatiger989/FT_Arabic_Whisper_V1_1"#"openai/whisper-large-v3-turbo"
+
+print(f"Using device: {device}")
+print(f"CUDA available: {torch.cuda.is_available()}")
+if torch.cuda.is_available():
+    print(f"GPU: {torch.cuda.get_device_name(0)}")
+
+# Model initialization with fallback for attention implementation
+try:
+    if use_flash_attn and torch.cuda.is_available():
+        model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            MODEL_NAME, 
+            torch_dtype=torch_dtype, 
+            low_cpu_mem_usage=True, 
+            use_safetensors=True, 
+            attn_implementation="flash_attention_2"
+        )
+    else:
+        model = AutoModelForSpeechSeq2Seq.from_pretrained(
+            MODEL_NAME, 
+            torch_dtype=torch_dtype, 
+            low_cpu_mem_usage=True, 
+            use_safetensors=True
+        )
+except Exception as e:
+    print(f"Error loading model with flash attention: {e}")
+    model = AutoModelForSpeechSeq2Seq.from_pretrained(
+        MODEL_NAME, 
+        torch_dtype=torch_dtype, 
+        low_cpu_mem_usage=True, 
+        use_safetensors=True
+    )
+
+model.to(device)
+
+processor = AutoProcessor.from_pretrained(MODEL_NAME)
+tokenizer = WhisperTokenizer.from_pretrained(MODEL_NAME)
+
+# Thread pool for processing audio
+executor = ThreadPoolExecutor(max_workers=4)
+
+class JambonzAudioBuffer:
+    def __init__(self, sample_rate=8000, chunk_duration=1.0):
+        self.sample_rate = sample_rate
+        self.chunk_duration = chunk_duration
+        self.chunk_samples = int(chunk_duration * sample_rate)
+        
+        self.buffer = np.array([], dtype=np.float32)
+        self.lock = threading.Lock()
+        self.total_audio = np.array([], dtype=np.float32)
+        
+        # Voice Activity Detection (simple energy-based)
+        self.silence_threshold = 0.01
+        self.min_speech_samples = int(0.3 * sample_rate)  # 300ms minimum speech
+        
+    def add_audio(self, audio_data):
+        with self.lock:
+            self.buffer = np.concatenate([self.buffer, audio_data])
+            self.total_audio = np.concatenate([self.total_audio, audio_data])
+    
+    def has_chunk_ready(self):
+        with self.lock:
+            return len(self.buffer) >= self.chunk_samples
+    
+    def is_speech(self, audio_chunk):
+        """Simple VAD based on energy"""
+        if len(audio_chunk) < self.min_speech_samples:
+            return False
+        energy = np.mean(np.abs(audio_chunk))
+        return energy > self.silence_threshold
+    
+    def get_chunk_for_processing(self):
+        """Get audio chunk for processing - but don't remove it from buffer for interim results"""
+        with self.lock:
+            if len(self.buffer) < self.chunk_samples:
+                return None
+            
+            # For interim results, we want to trigger processing but keep accumulating audio
+            # So we just return a signal that we have enough audio, but don't consume it
+            return np.array([1])  # Return a dummy array to signal chunk is ready
+    
+    def get_all_audio(self):
+        """Get all accumulated audio for final transcription"""
+        with self.lock:
+            return self.total_audio.copy()
+    
+    def clear(self):
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+
+def linear16_to_audio(audio_bytes, sample_rate=8000):
+    """Convert LINEAR16 PCM bytes to numpy array (jambonz format)"""
+    try:
+        # jambonz sends LINEAR16 PCM at 8kHz
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16)
+        # Convert to float32 and normalize
+        audio_array = audio_array.astype(np.float32) / 32768.0
+        return audio_array
+    except Exception as e:
+        logger.error(f"Error converting LINEAR16 to audio: {e}")
+        return np.array([], dtype=np.float32)
+
+def resample_audio(audio_data, source_rate, target_rate):
+    """Simple resampling from 8kHz to 16kHz for Whisper"""
+    if source_rate == target_rate:
+        return audio_data
+    
+    # Simple linear interpolation resampling
+    ratio = target_rate / source_rate
+    indices = np.arange(0, len(audio_data), 1/ratio)
+    indices = indices[indices < len(audio_data)]
+    resampled = np.interp(indices, np.arange(len(audio_data)), audio_data)
+    
+    # Ensure proper float32 dtype for consistency
+    return resampled.astype(np.float32)
+def transcribe_chunk_direct(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio chunk using model's generate method directly"""
+    try:
+        if len(audio_data) == 0:
+            return ""
+
+        # Resample from 8kHz to 16kHz for Whisper
+        resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+
+        # --- Denoiser added ---
+        resampled_audio = denoise_audio(resampled_audio, sample_rate=target_sample_rate)
+
+        # Ensure minimum length for Whisper
+        min_samples = int(0.1 * target_sample_rate)  # 100ms minimum
+        if len(resampled_audio) < min_samples:
+            return ""
+
+        start_time = time.time()
+
+        # Prepare input features
+        input_features = processor(
+            resampled_audio,
+            sampling_rate=target_sample_rate,
+            return_tensors="pt"
+        ).input_features
+
+        input_features = input_features.to(device=device, dtype=torch_dtype)
+
+        attention_mask = torch.ones(
+            input_features.shape[:-1],
+            dtype=torch.long,
+            device=device
+        )
+
+        with torch.no_grad():
+            predicted_ids = model.generate(
+                input_features,
+                attention_mask=attention_mask,
+                max_new_tokens=128,
+                do_sample=False,
+                temperature=0.0,
+                num_beams=1,
+                language="ar",
+                task="transcribe",
+                pad_token_id=tokenizer.pad_token_id,
+                eos_token_id=tokenizer.eos_token_id
+            )
+
+        transcription = tokenizer.batch_decode(
+            predicted_ids,
+            skip_special_tokens=True
+        )[0].strip()
+
+        end_time = time.time()
+
+        logger.info(f"Direct transcription completed in {end_time - start_time:.2f}s: '{transcription}'")
+        return transcription
+
+    except Exception as e:
+        logger.error(f"Error during direct transcription: {e}")
+        return ""
+# def transcribe_chunk_direct(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+#     """Transcribe audio chunk using model's generate method directly"""
+#     try:
+#         if len(audio_data) == 0:
+#             return ""
+        
+#         # Resample from 8kHz to 16kHz for Whisper
+#         resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        
+#         # Ensure minimum length for Whisper
+#         min_samples = int(0.1 * target_sample_rate)  # 100ms minimum
+#         if len(resampled_audio) < min_samples:
+#             return ""
+        
+#         start_time = time.time()
+        
+#         # Prepare input features with proper dtype
+#         input_features = processor(
+#             resampled_audio, 
+#             sampling_rate=target_sample_rate, 
+#             return_tensors="pt"
+#         ).input_features
+        
+#         # Ensure correct dtype and device
+#         input_features = input_features.to(device=device, dtype=torch_dtype)
+        
+#         # Create attention mask to avoid warnings
+#         attention_mask = torch.ones(
+#             input_features.shape[:-1], 
+#             dtype=torch.long, 
+#             device=device
+#         )
+        
+#         # Generate transcription using model directly
+#         with torch.no_grad():
+#             predicted_ids = model.generate(
+#                 input_features,
+#                 attention_mask=attention_mask,
+#                 max_new_tokens=128,
+#                 do_sample=False,
+#                 temperature=0.0,
+#                 num_beams=1,
+#                 language="ar",
+#                 task="transcribe",
+#                 pad_token_id=tokenizer.pad_token_id,
+#                 eos_token_id=tokenizer.eos_token_id
+#             )
+        
+#         # Decode the transcription
+#         transcription = tokenizer.batch_decode(
+#             predicted_ids, 
+#             skip_special_tokens=True
+#         )[0].strip()
+        
+#         end_time = time.time()
+        
+#         logger.info(f"Direct transcription completed in {end_time - start_time:.2f}s: '{transcription}'")
+#         return transcription
+        
+#     except Exception as e:
+#         logger.error(f"Error during direct transcription: {e}")
+#         return ""
+
+class JambonzSTTHandler:
+    def __init__(self, websocket):
+        self.websocket = websocket
+        self.audio_buffer = None
+        self.config = {}
+        self.running = True
+        self.transcription_task = None
+        self.full_transcript = ""
+        self.last_partial = ""
+        
+        # Auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.silence_timeout = 1.5  # 3 seconds of silence to trigger final
+        self.min_interim_count = 1  # Minimum interim results before considering final
+        self.auto_final_task = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        
+    async def start_processing(self, start_message):
+        """Initialize with start message from jambonz"""
+        self.config = {
+            "language": start_message.get("language", "ar-EG"),
+            "format": start_message.get("format", "raw"),
+            "encoding": start_message.get("encoding", "LINEAR16"),
+            "sample_rate": start_message.get("sampleRateHz", 8000),
+            "interim_results": start_message.get("interimResults", True),
+            "options": start_message.get("options", {})
+        }
+        
+        logger.info(f"STT session started with config: {self.config}")
+        
+        # Initialize audio buffer
+        self.audio_buffer = JambonzAudioBuffer(
+            sample_rate=self.config["sample_rate"],
+            chunk_duration=1.0  # Process every 1 second
+        )
+        
+        # Reset auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        
+        # Start background transcription task
+        self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+        # Start auto-final detection task
+        self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+    
+    async def stop_processing(self):
+        """Stop processing and send final transcription"""
+        self.running = False
+        
+        # Cancel background tasks
+        if self.transcription_task:
+            self.transcription_task.cancel()
+            try:
+                await self.transcription_task
+            except asyncio.CancelledError:
+                pass
+        
+        if self.auto_final_task:
+            self.auto_final_task.cancel()
+            try:
+                await self.auto_final_task
+            except asyncio.CancelledError:
+                pass
+        
+        # Send final transcription if not already sent
+        if not self.final_sent and self.accumulated_transcript.strip():
+            await self.send_transcription(self.accumulated_transcript, is_final=True)
+        
+        # Also process any remaining audio for comprehensive final transcription
+        if self.audio_buffer:
+            all_audio = self.audio_buffer.get_all_audio()
+            if len(all_audio) > 0 and not self.final_sent:
+                loop = asyncio.get_event_loop()
+                final_transcription = await loop.run_in_executor(
+                    executor, 
+                    transcribe_chunk_direct, 
+                    all_audio,
+                    self.config["sample_rate"]
+                )
+                
+                if final_transcription.strip():
+                    # Send comprehensive final transcription
+                    await self.send_transcription(final_transcription, is_final=True)
+        
+        logger.info("STT session ended")
+    
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer"""
+        if self.audio_buffer:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+    
+    async def _process_audio_chunks(self):
+        """Process audio chunks for interim results"""
+        while self.running and self.config.get("interim_results", False):
+            try:
+                if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+                    chunk_signal = self.audio_buffer.get_chunk_for_processing()
+                    if chunk_signal is not None:
+                        # Get all accumulated audio so far for complete transcription
+                        all_audio = self.audio_buffer.get_all_audio()
+                        
+                        # Only process if we have actual speech content
+                        if len(all_audio) > 0 and self.audio_buffer.is_speech(all_audio[-self.audio_buffer.chunk_samples:]):
+                            # Run transcription on all accumulated audio
+                            loop = asyncio.get_event_loop()
+                            transcription = await loop.run_in_executor(
+                                executor, 
+                                transcribe_chunk_direct, 
+                                all_audio,
+                                self.config["sample_rate"]
+                            )
+                            
+                            if transcription.strip() and transcription != self.last_partial:
+                                self.last_partial = transcription
+                                self.accumulated_transcript = transcription  # Update accumulated transcript
+                                self.interim_count += 1
+                                self.last_interim_time = time.time()
+                                
+                                # Send interim result
+                                await self.send_transcription(transcription, is_final=False)
+                                
+                                logger.info(f"Interim #{self.interim_count}: '{transcription}'")
+                
+                # Small delay to prevent excessive processing
+                await asyncio.sleep(0.1)
+                
+            except Exception as e:
+                logger.error(f"Error in chunk processing: {e}")
+                await asyncio.sleep(1)
+    
+    async def _monitor_for_auto_final(self):
+        """Monitor for auto-final conditions: 3 seconds silence after 3+ interim results"""
+        while self.running:
+            try:
+                current_time = time.time()
+                
+                # Check if we should send auto-final transcription
+                if (self.interim_count >= self.min_interim_count and 
+                    self.last_interim_time is not None and 
+                    (current_time - self.last_interim_time) >= self.silence_timeout and
+                    not self.final_sent and
+                    self.accumulated_transcript.strip()):
+                    
+                    logger.info(f"Auto-final triggered: {self.interim_count} interim results, "
+                              f"{current_time - self.last_interim_time:.1f}s silence")
+                    
+                    # Send the accumulated transcript as final
+                    await self.send_transcription(self.accumulated_transcript, is_final=True)
+                    self.final_sent = True
+                    
+                    # Reset counters for potential next utterance
+                    self.interim_count = 0
+                    self.last_interim_time = None
+                    self.accumulated_transcript = ""
+                
+                # Check every 0.5 seconds
+                await asyncio.sleep(0.5)
+                
+            except Exception as e:
+                logger.error(f"Error in auto-final monitoring: {e}")
+                await asyncio.sleep(1)
+    
+    # async def send_transcription(self, text, is_final=False, confidence=0.9):
+    #     """Send transcription in jambonz format with Arabic number conversion"""
+    #     try:
+    #         # Convert Arabic numbers to digits before sending
+    #         original_text = text
+    #         converted_text = convert_arabic_numbers_in_sentence(text)
+            
+    #         # Log the conversion if numbers were found and converted
+    #         if original_text != converted_text:
+    #             logger.info(f"Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+            
+    #         message = {
+    #             "type": "transcription",
+    #             "is_final": is_final,
+    #             "alternatives": [
+    #                 {
+    #                     "transcript": converted_text,
+    #                     "confidence": confidence
+    #                 }
+    #             ],
+    #             "language": self.config.get("language", "ar-EG"),
+    #             "channel": 1
+    #         }
+            
+    #         await self.websocket.send(json.dumps(message))
+    #         logger.info(f"Sent {'FINAL' if is_final else 'interim'} transcription: '{converted_text}'")
+            
+    #         if is_final:
+    #             self.final_sent = True
+            
+    #     except Exception as e:
+    #         logger.error(f"Error sending transcription: {e}")
+    
+
+
+    async def send_transcription(self, text, is_final=False, confidence=0.9):
+        """Send transcription in jambonz format with Arabic number conversion, only for final results"""
+        try:
+            if not is_final:
+                # Do nothing for interim results
+                logger.debug("Skipping interim transcription (not final).")
+                return
+
+            # Convert Arabic numbers only for final transcripts
+            original_text = text
+            converted_text = convert_arabic_numbers_in_sentence(text)
+
+            # Log the conversion if numbers were found and converted
+            if original_text != converted_text:
+                logger.info(f"Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+
+            message = {
+                "type": "transcription",
+                "is_final": True,
+                "alternatives": [
+                    {
+                        "transcript": original_text,#converted_text,
+                        "confidence": confidence
+                    }
+                ],
+                "language": self.config.get("language", "ar-EG"),
+                "channel": 1
+            }
+
+            # Send only final messages
+            await self.websocket.send(json.dumps(message))
+            logger.info(f"Sent FINAL transcription: '{converted_text}'")
+
+            self.final_sent = True
+
+        except Exception as e:
+            logger.error(f"Error sending transcription: {e}")
+
+
+
+
+    async def send_error(self, error_message):
+        """Send error message in jambonz format"""
+        try:
+            message = {
+                "type": "error",
+                "error": error_message
+            }
+            await self.websocket.send(json.dumps(message))
+            logger.error(f"Sent error: {error_message}")
+        except Exception as e:
+            logger.error(f"Error sending error message: {e}")
+
+async def handle_jambonz_websocket(websocket):
+    """Handle jambonz WebSocket connections"""
+    
+    client_id = f"jambonz_{id(websocket)}"
+    logger.info(f"New jambonz connection: {client_id}")
+    
+    handler = JambonzSTTHandler(websocket)
+    
+    try:
+        async for message in websocket:
+            try:
+                if isinstance(message, str):
+                    # Handle JSON control messages
+                    data = json.loads(message)
+                    message_type = data.get("type")
+                    
+                    if message_type == "start":
+                        logger.info(f"Received start message: {data}")
+                        await handler.start_processing(data)
+                        
+                    elif message_type == "stop":
+                        logger.info("Received stop message")
+                        await handler.stop_processing()
+                        # Close websocket after final transcription
+                        await websocket.close(code=1000, reason="Session completed")
+                        break
+                        
+                    else:
+                        logger.warning(f"Unknown message type: {message_type}")
+                        await handler.send_error(f"Unknown message type: {message_type}")
+                
+                else:
+                    # Handle binary audio data (LINEAR16 PCM)
+                    if handler.audio_buffer is None:
+                        await handler.send_error("Received audio before start message")
+                        continue
+                    
+                    await handler.add_audio_data(message)
+            
+            except json.JSONDecodeError as e:
+                logger.error(f"JSON decode error: {e}")
+                await handler.send_error(f"Invalid JSON: {str(e)}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+                await handler.send_error(f"Processing error: {str(e)}")
+    
+    except websockets.exceptions.ConnectionClosed:
+        logger.info(f"jambonz connection closed: {client_id}")
+    except Exception as e:
+        logger.error(f"jambonz WebSocket error: {e}")
+        try:
+            await handler.send_error(str(e))
+        except:
+            pass
+    finally:
+        if handler.running:
+            await handler.stop_processing()
+        logger.info(f"jambonz connection ended: {client_id}")
+
+async def main():
+    """Start the jambonz STT WebSocket server"""
+    logger.info("Starting Jambonz Custom STT WebSocket server on port 3006...")
+    
+    # Start WebSocket server
+    server = await websockets.serve(
+        handle_jambonz_websocket,
+        "0.0.0.0", 
+        3006,
+        ping_interval=20,
+        ping_timeout=10,
+        close_timeout=10
+    )
+    
+    logger.info("Jambonz Custom STT WebSocket server started on ws://0.0.0.0:3006")
+    logger.info("Ready to handle jambonz STT requests")
+    logger.info("- Expects LINEAR16 PCM audio at 8kHz")
+    logger.info("- Supports interim results with auto-final detection")
+    logger.info("- Auto-final: 3+ interim results + 1.3s silence")
+    logger.info("- Resamples to 16kHz for Whisper processing")
+    logger.info("- Converts Arabic numbers to digits before sending")
+    
+    # Wait for the server to close
+    await server.wait_closed()
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print("Jambonz Custom STT Server with Whisper + Arabic Numbers")
+    print("=" * 60)
+    print(f"Model: {MODEL_NAME}")
+    print(f"Device: {device}")
+    print("WebSocket Port: 3006")
+    print("Protocol: jambonz STT API")
+    print("Audio Format: LINEAR16 PCM @ 8kHz")
+    print("Auto-Final: 2+ speech activities + 1.3s silence")
+    print("Arabic Numbers: Converted to digits in FINAL transcriptions only")
+    print("Interim Results: DISABLED (final transcription only)")
+    if arabic_numbers_available:
+        print("✓ pyarabic library available for number conversion")
+    else:
+        print("✗ pyarabic library not available - install with: pip install pyarabic")
+    print("=" * 60)
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nShutting down server...")
+    except Exception as e:
+        print(f"Server error: {e}")

asr_websocket_client.html

@@ -0,0 +1,606 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>ASR WebSocket Testing Client</title>
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+
+        body {
+            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            min-height: 100vh;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            padding: 20px;
+        }
+
+        .container {
+            background: rgba(255, 255, 255, 0.95);
+            backdrop-filter: blur(10px);
+            border-radius: 20px;
+            padding: 40px;
+            box-shadow: 0 20px 40px rgba(0, 0, 0, 0.1);
+            max-width: 600px;
+            width: 100%;
+            border: 1px solid rgba(255, 255, 255, 0.2);
+        }
+
+        .header {
+            text-align: center;
+            margin-bottom: 30px;
+        }
+
+        .header h1 {
+            color: #333;
+            font-size: 2.5em;
+            font-weight: 300;
+            margin-bottom: 10px;
+        }
+
+        .header p {
+            color: #666;
+            font-size: 1.1em;
+        }
+
+        .connection-section {
+            margin-bottom: 30px;
+        }
+
+        .input-group {
+            margin-bottom: 20px;
+        }
+
+        .input-group label {
+            display: block;
+            margin-bottom: 8px;
+            color: #333;
+            font-weight: 500;
+        }
+
+        .input-group input {
+            width: 100%;
+            padding: 12px 16px;
+            border: 2px solid #e1e5e9;
+            border-radius: 10px;
+            font-size: 16px;
+            transition: all 0.3s ease;
+            background: rgba(255, 255, 255, 0.8);
+        }
+
+        .input-group input:focus {
+            outline: none;
+            border-color: #667eea;
+            box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1);
+        }
+
+        .btn {
+            padding: 12px 24px;
+            border: none;
+            border-radius: 10px;
+            font-size: 16px;
+            font-weight: 500;
+            cursor: pointer;
+            transition: all 0.3s ease;
+            text-transform: uppercase;
+            letter-spacing: 0.5px;
+        }
+
+        .btn:disabled {
+            opacity: 0.6;
+            cursor: not-allowed;
+        }
+
+        .btn-connect {
+            background: linear-gradient(135deg, #4CAF50, #45a049);
+            color: white;
+            width: 100%;
+        }
+
+        .btn-connect:hover:not(:disabled) {
+            transform: translateY(-2px);
+            box-shadow: 0 5px 15px rgba(76, 175, 80, 0.3);
+        }
+
+        .btn-disconnect {
+            background: linear-gradient(135deg, #f44336, #da190b);
+            color: white;
+            width: 100%;
+        }
+
+        .btn-disconnect:hover:not(:disabled) {
+            transform: translateY(-2px);
+            box-shadow: 0 5px 15px rgba(244, 67, 54, 0.3);
+        }
+
+        .audio-controls {
+            display: flex;
+            justify-content: center;
+            gap: 20px;
+            margin: 30px 0;
+        }
+
+        .btn-mic {
+            background: linear-gradient(135deg, #2196F3, #1976D2);
+            color: white;
+            width: 80px;
+            height: 80px;
+            border-radius: 50%;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            font-size: 24px;
+        }
+
+        .btn-mic:hover:not(:disabled) {
+            transform: scale(1.1);
+            box-shadow: 0 10px 25px rgba(33, 150, 243, 0.3);
+        }
+
+        .btn-mic.recording {
+            background: linear-gradient(135deg, #f44336, #da190b);
+            animation: pulse 1.5s infinite;
+        }
+
+        .btn-stop {
+            background: linear-gradient(135deg, #FF9800, #F57C00);
+            color: white;
+            width: 80px;
+            height: 80px;
+            border-radius: 50%;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            font-size: 24px;
+        }
+
+        .btn-stop:hover:not(:disabled) {
+            transform: scale(1.1);
+            box-shadow: 0 10px 25px rgba(255, 152, 0, 0.3);
+        }
+
+        @keyframes pulse {
+            0% { transform: scale(1); }
+            50% { transform: scale(1.05); }
+            100% { transform: scale(1); }
+        }
+
+        .status {
+            text-align: center;
+            margin: 20px 0;
+            padding: 12px;
+            border-radius: 10px;
+            font-weight: 500;
+        }
+
+        .status.connected {
+            background: rgba(76, 175, 80, 0.1);
+            color: #4CAF50;
+            border: 1px solid rgba(76, 175, 80, 0.3);
+        }
+
+        .status.disconnected {
+            background: rgba(244, 67, 54, 0.1);
+            color: #f44336;
+            border: 1px solid rgba(244, 67, 54, 0.3);
+        }
+
+        .status.recording {
+            background: rgba(33, 150, 243, 0.1);
+            color: #2196F3;
+            border: 1px solid rgba(33, 150, 243, 0.3);
+        }
+
+        .response-section {
+            margin-top: 30px;
+        }
+
+        .response-box {
+            background: rgba(0, 0, 0, 0.05);
+            border-radius: 10px;
+            padding: 20px;
+            min-height: 120px;
+            border: 1px solid rgba(0, 0, 0, 0.1);
+            font-family: 'Courier New', monospace;
+            white-space: pre-wrap;
+            word-wrap: break-word;
+        }
+
+        .loading {
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            color: #666;
+        }
+
+        .loading::after {
+            content: '';
+            width: 20px;
+            height: 20px;
+            border: 2px solid #f3f3f3;
+            border-top: 2px solid #667eea;
+            border-radius: 50%;
+            animation: spin 1s linear infinite;
+            margin-left: 10px;
+        }
+
+        @keyframes spin {
+            0% { transform: rotate(0deg); }
+            100% { transform: rotate(360deg); }
+        }
+
+        .audio-visualizer {
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            height: 40px;
+            margin: 10px 0;
+        }
+
+        .bar {
+            width: 3px;
+            height: 10px;
+            background: #667eea;
+            margin: 0 1px;
+            border-radius: 2px;
+            animation: wave 1s ease-in-out infinite;
+        }
+
+        .bar:nth-child(2) { animation-delay: 0.1s; }
+        .bar:nth-child(3) { animation-delay: 0.2s; }
+        .bar:nth-child(4) { animation-delay: 0.3s; }
+        .bar:nth-child(5) { animation-delay: 0.4s; }
+
+        @keyframes wave {
+            0%, 100% { height: 10px; }
+            50% { height: 30px; }
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="header">
+            <h1>🎤 ASR Tester</h1>
+            <p>WebSocket-based Speech Recognition Testing</p>
+        </div>
+
+        <div class="connection-section">
+            <div class="input-group">
+                <label for="websocketUrl">WebSocket URL:</label>
+                <input type="text" id="websocketUrl" value="ws://52.59.169.24:3015" placeholder="ws://localhost:5005/url">
+            </div>
+            <button id="connectBtn" class="btn btn-connect">Connect</button>
+            <button id="disconnectBtn" class="btn btn-disconnect" style="display: none;">Disconnect</button>
+        </div>
+
+        <div id="status" class="status disconnected">Disconnected</div>
+
+        <div class="audio-controls">
+            <button id="micBtn" class="btn btn-mic" disabled title="Start Recording">🎤</button>
+            <button id="stopBtn" class="btn btn-stop" disabled title="Stop Recording">⏹️</button>
+        </div>
+
+        <div id="visualizer" class="audio-visualizer" style="display: none;">
+            <div class="bar"></div>
+            <div class="bar"></div>
+            <div class="bar"></div>
+            <div class="bar"></div>
+            <div class="bar"></div>
+        </div>
+
+        <div class="response-section">
+            <h3>ASR Response:</h3>
+            <div id="responseBox" class="response-box">Waiting for audio input...</div>
+        </div>
+    </div>
+
+    <script>
+        class JambonzASRClient {
+            constructor() {
+                this.websocket = null;
+                this.audioContext = null;
+                this.mediaRecorder = null;
+                this.audioStream = null;
+                this.processor = null;
+                this.isRecording = false;
+                this.isConnected = false;
+                
+                this.initializeElements();
+                this.attachEventListeners();
+            }
+
+            initializeElements() {
+                this.elements = {
+                    websocketUrl: document.getElementById('websocketUrl'),
+                    connectBtn: document.getElementById('connectBtn'),
+                    disconnectBtn: document.getElementById('disconnectBtn'),
+                    micBtn: document.getElementById('micBtn'),
+                    stopBtn: document.getElementById('stopBtn'),
+                    status: document.getElementById('status'),
+                    responseBox: document.getElementById('responseBox'),
+                    visualizer: document.getElementById('visualizer')
+                };
+            }
+
+            attachEventListeners() {
+                this.elements.connectBtn.addEventListener('click', () => this.connect());
+                this.elements.disconnectBtn.addEventListener('click', () => this.disconnect());
+                this.elements.micBtn.addEventListener('click', () => this.startRecording());
+                this.elements.stopBtn.addEventListener('click', () => this.stopRecording());
+            }
+
+            updateStatus(message, type) {
+                this.elements.status.textContent = message;
+                this.elements.status.className = `status ${type}`;
+            }
+
+            async connect() {
+                const url = this.elements.websocketUrl.value.trim();
+                if (!url) {
+                    alert('Please enter a WebSocket URL');
+                    return;
+                }
+
+                try {
+                    this.updateStatus('Connecting...', 'disconnected');
+                    this.elements.connectBtn.disabled = true;
+
+                    this.websocket = new WebSocket(url);
+                    this.websocket.binaryType = 'arraybuffer';
+
+                    this.websocket.onopen = () => {
+                        this.isConnected = true;
+                        this.updateStatus('Connected - Ready for Jambonz Protocol', 'connected');
+                        this.elements.connectBtn.style.display = 'none';
+                        this.elements.disconnectBtn.style.display = 'block';
+                        this.elements.micBtn.disabled = false;
+                        this.elements.responseBox.textContent = 'Connected. Ready to start ASR session...';
+                    };
+
+                    this.websocket.onmessage = (event) => {
+                        if (typeof event.data === 'string') {
+                            try {
+                                const response = JSON.parse(event.data);
+                                this.displayResponse('JSON Control Message', response);
+                            } catch (e) {
+                                this.displayResponse('Text Message', event.data);
+                            }
+                        } else {
+                            // Binary data (should not happen in normal Jambonz flow from server)
+                            this.displayResponse('Binary Message', `Received binary data: ${event.data.byteLength} bytes`);
+                        }
+                    };
+
+                    this.websocket.onerror = (error) => {
+                        console.error('WebSocket error:', error);
+                        this.updateStatus('Connection Error', 'disconnected');
+                        this.resetConnection();
+                    };
+
+                    this.websocket.onclose = (event) => {
+                        this.isConnected = false;
+                        this.updateStatus(`Disconnected (Code: ${event.code})`, 'disconnected');
+                        this.resetConnection();
+                        this.displayResponse('Connection Closed', `WebSocket closed with code: ${event.code}, reason: ${event.reason || 'No reason provided'}`);
+                    };
+
+                } catch (error) {
+                    console.error('Connection failed:', error);
+                    this.updateStatus('Connection Failed', 'disconnected');
+                    this.resetConnection();
+                }
+            }
+
+            disconnect() {
+                if (this.isRecording) {
+                    this.stopRecording();
+                }
+                if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
+                    this.websocket.close(1000, 'Client disconnect');
+                }
+                this.resetConnection();
+            }
+
+            resetConnection() {
+                this.isConnected = false;
+                this.elements.connectBtn.disabled = false;
+                this.elements.connectBtn.style.display = 'block';
+                this.elements.disconnectBtn.style.display = 'none';
+                this.elements.micBtn.disabled = true;
+                this.elements.stopBtn.disabled = true;
+                this.stopRecording();
+            }
+
+            // Convert Float32Array to Int16Array (LINEAR16 PCM)
+            floatTo16BitPCM(float32Array) {
+                const int16Array = new Int16Array(float32Array.length);
+                for (let i = 0; i < float32Array.length; i++) {
+                    const clipped = Math.max(-1, Math.min(1, float32Array[i]));
+                    int16Array[i] = clipped * 0x7FFF;
+                }
+                return int16Array;
+            }
+
+            // Resample audio from source sample rate to 8kHz
+            resampleTo8kHz(audioBuffer, sourceSampleRate) {
+                const targetSampleRate = 8000;
+                const ratio = sourceSampleRate / targetSampleRate;
+                const targetLength = Math.round(audioBuffer.length / ratio);
+                const resampled = new Float32Array(targetLength);
+                
+                for (let i = 0; i < targetLength; i++) {
+                    const sourceIndex = i * ratio;
+                    const sourceIndexFloor = Math.floor(sourceIndex);
+                    const sourceIndexCeil = Math.min(sourceIndexFloor + 1, audioBuffer.length - 1);
+                    const weight = sourceIndex - sourceIndexFloor;
+                    
+                    resampled[i] = audioBuffer[sourceIndexFloor] * (1 - weight) + 
+                                   audioBuffer[sourceIndexCeil] * weight;
+                }
+                
+                return resampled;
+            }
+
+            async startRecording() {
+                if (!this.isConnected) {
+                    alert('Please connect to WebSocket first');
+                    return;
+                }
+
+                try {
+                    // Initialize audio context
+                    this.audioContext = new (window.AudioContext || window.webkitAudioContext)({
+                        sampleRate: 44100 // Let browser choose, we'll resample
+                    });
+
+                    // Get microphone stream
+                    this.audioStream = await navigator.mediaDevices.getUserMedia({ 
+                        audio: {
+                            echoCancellation: false,  // Jambonz typically handles this
+                            noiseSuppression: false,  // Jambonz typically handles this
+                            autoGainControl: false,   // Jambonz typically handles this
+                            channelCount: 1           // Mono audio
+                        } 
+                    });
+
+                    const source = this.audioContext.createMediaStreamSource(this.audioStream);
+
+                    // Create ScriptProcessorNode for audio processing
+                    // Note: ScriptProcessorNode is deprecated but still widely supported
+                    // In production, consider using AudioWorklet
+                    this.processor = this.audioContext.createScriptProcessor(4096, 1, 1);
+                    
+                    this.processor.onaudioprocess = (event) => {
+                        if (!this.isRecording || !this.websocket || this.websocket.readyState !== WebSocket.OPEN) {
+                            return;
+                        }
+
+                        const inputBuffer = event.inputBuffer;
+                        const audioData = inputBuffer.getChannelData(0); // Get mono channel
+                        
+                        // Resample to 8kHz
+                        const resampled = this.resampleTo8kHz(audioData, this.audioContext.sampleRate);
+                        
+                        // Convert to LINEAR16 PCM
+                        const pcmData = this.floatTo16BitPCM(resampled);
+                        
+                        // Send binary audio data
+                        this.websocket.send(pcmData.buffer);
+                    };
+
+                    // Connect audio nodes
+                    source.connect(this.processor);
+                    this.processor.connect(this.audioContext.destination);
+
+                    // Send Jambonz START control message
+                    const startMessage = {
+                        type: "start",
+                        language: "en-US",
+                        format: "raw",
+                        encoding: "LINEAR16",
+                        interimResults: true,
+                        sampleRateHz: 8000,
+                        options: {
+                            callSid: Date.now().toString()
+                        }
+                    };
+
+                    this.websocket.send(JSON.stringify(startMessage));
+                    this.displayResponse('Sent START Message', startMessage);
+
+                    this.isRecording = true;
+
+                    // Update UI
+                    this.elements.micBtn.classList.add('recording');
+                    this.elements.micBtn.disabled = true;
+                    this.elements.stopBtn.disabled = false;
+                    this.elements.visualizer.style.display = 'flex';
+                    this.updateStatus('Recording - Sending LINEAR16 PCM @ 8kHz', 'recording');
+
+                } catch (error) {
+                    console.error('Failed to start recording:', error);
+                    alert('Failed to access microphone. Please check permissions.');
+                    this.stopRecording();
+                }
+            }
+
+            stopRecording() {
+                if (this.isRecording) {
+                    this.isRecording = false;
+
+                    // Send Jambonz STOP control message
+                    if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
+                        const stopMessage = {
+                            type: "stop"
+                        };
+                        this.websocket.send(JSON.stringify(stopMessage));
+                        this.displayResponse('Sent STOP Message', stopMessage);
+                    }
+                }
+
+                // Clean up audio resources
+                if (this.processor) {
+                    this.processor.disconnect();
+                    this.processor = null;
+                }
+
+                if (this.audioContext) {
+                    this.audioContext.close().then(() => {
+                        this.audioContext = null;
+                    });
+                }
+
+                if (this.audioStream) {
+                    this.audioStream.getTracks().forEach(track => track.stop());
+                    this.audioStream = null;
+                }
+
+                // Update UI
+                this.elements.micBtn.classList.remove('recording');
+                this.elements.micBtn.disabled = false;
+                this.elements.stopBtn.disabled = true;
+                this.elements.visualizer.style.display = 'none';
+                
+                if (this.isConnected) {
+                    this.updateStatus('Connected - Waiting for final transcript...', 'connected');
+                }
+            }
+
+            displayResponse(messageType, response) {
+                const responseBox = this.elements.responseBox;
+                const timestamp = new Date().toLocaleTimeString();
+                
+                let content = `<strong>[${timestamp}] ${messageType}:</strong>\n`;
+                
+                if (typeof response === 'object') {
+                    content += JSON.stringify(response, null, 2);
+                } else {
+                    content += response;
+                }
+                
+                // Append to existing content
+                if (responseBox.innerHTML.includes('Connected. Ready to start ASR session...') || 
+                    responseBox.innerHTML.includes('Processing audio...')) {
+                    responseBox.innerHTML = content;
+                } else {
+                    responseBox.innerHTML += '\n\n' + content;
+                }
+                
+                // Auto-scroll to bottom
+                responseBox.scrollTop = responseBox.scrollHeight;
+            }
+        }
+
+        // Initialize the client when page loads
+        document.addEventListener('DOMContentLoaded', () => {
+            new JambonzASRClient();
+        });
+    </script>
+</body>
+</html>

best_nemo_whisper_jambonz.py

@@ -0,0 +1,1338 @@
+import torch
+import asyncio
+import websockets
+import json
+import threading
+import numpy as np
+import logging
+import time
+import tempfile
+import os
+import re
+from concurrent.futures import ThreadPoolExecutor
+import subprocess
+import struct
+
+# NeMo imports
+import nemo.collections.asr as nemo_asr
+import soundfile as sf
+
+# Whisper imports
+# from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, WhisperTokenizer, pipeline
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
+
+
+# Arabic number conversion imports for Whisper
+try:
+    from pyarabic.number import text2number
+    arabic_numbers_available = True
+    print("✓ pyarabic library available for Whisper number conversion")
+except ImportError:
+    arabic_numbers_available = False
+    print("✗ pyarabic not available - install with: pip install pyarabic")
+    print("Arabic numbers will not be converted to digits for Whisper")
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# ===== NeMo Arabic number mapping =====
+arabic_numbers_nemo = {
+    # Basic digits
+    "سفر": "0", "فيرو": "0", "هيرو": "0","صفر": "0", "زيرو": "0", "٠": "0","زيو": "0","زير": "0","زير": "0","زر": "0","زروا": "0","زرا": "0","زيره ": "0","زرو ": "0",
+    "واحد": "1", "واحدة": "1", "١": "1",
+    "اتنين": "2", "اثنين": "2", "إثنين": "2", "اثنان": "2", "إثنان": "2", "٢": "2",
+    "تلاتة": "3", "ثلاثة": "3", "٣": "3","تلاته": "3","ثلاثه": "3","ثلاثا": "3","تلاتا": "3",
+    "اربعة": "4", "أربعة": "4", "٤": "4","اربعه": "4","أربعه": "4","أربع": "4","اربع": "4","اربعا": "4","أربعا": "4",
+    "خمسة": "5", "خمسه": "5", "٥": "5", "خمس": "5", "خمسا": "5",
+    "ستة": "6", "سته": "6", "٦": "6", "ست": "6", "ستّا": "6", "ستةً": "6",
+    "سبعة": "7", "سبعه": "7", "٧": "7", "سبع": "7", "سبعا": "7",
+    "ثمانية": "8", "ثمانيه": "8", "٨": "8", "ثمان": "8", "ثمنية": "8", "ثمنيه": "8", "ثمانيا": "8", "ثمن": "8",
+    "تسعة": "9", "تسعه": "9", "٩": "9", "تسع": "9", "تسعا": "9",
+    
+    # Teens
+    "عشرة": "10", "١٠": "10",
+    "حداشر": "11", "احد عشر": "11","احداشر": "11",
+    "اتناشر": "12", "اثنا عشر": "12",
+    "تلتاشر": "13", "ثلاثة عشر": "13",
+    "اربعتاشر": "14", "أربعة عشر": "14",
+    "خمستاشر": "15", "خمسة عشر": "15",
+    "ستاشر": "16", "ستة عشر": "16",
+    "سبعتاشر": "17", "سبعة عشر": "17",
+    "طمنتاشر": "18", "ثمانية عشر": "18",
+    "تسعتاشر": "19", "تسعة عشر": "19",
+    
+    # Tens
+    "عشرين": "20", "٢٠": "20",
+    "تلاتين": "30", "ثلاثين": "30", "٣٠": "30",
+    "اربعين": "40", "أربعين": "40", "٤٠": "40",
+    "خمسين": "50", "٥٠": "50",
+    "ستين": "60", "٦٠": "60",
+    "سبعين": "70", "٧٠": "70",
+    "تمانين": "80", "ثمانين": "80", "٨٠": "80","تمانون": "80","ثمانون": "80",
+    "تسعين": "90", "٩٠": "90",
+    
+    # Hundreds
+    "مية": "100", "مائة": "100", "مئة": "100", "١٠٠": "100",
+    "ميتين": "200", "مائتين": "200",
+    "تلاتمية": "300", "ثلاثمائة": "300",
+    "اربعمية": "400", "أربعمائة": "400",
+    "خمسمية": "500", "خمسمائة": "500",
+    "ستمية": "600", "ستمائة": "600",
+    "سبعمية": "700", "سبعمائة": "700",
+    "تمانمية": "800", "ثمانمائة": "800",
+    "تسعمية": "900", "تسعمائة": "900",
+    
+    # Thousands
+    "ألف": "1000", "الف": "1000", "١٠٠٠": "1000",
+    "ألفين": "2000", "الفين": "2000",
+    "تلات تلاف": "3000", "ثلاثة آلاف": "3000",
+    "اربعة آلاف": "4000", "أربعة آلاف": "4000",
+    "خمسة آلاف": "5000",
+    "ستة آلاف": "6000",
+    "سبعة آلاف": "7000",
+    "تمانية آلاف": "8000", "ثمانية آلاف": "8000",
+    "تسعة آلاف": "9000",
+    
+    # Large numbers
+    "عشرة آلاف": "10000",
+    "مية ألف": "100000", "مائة ألف": "100000",
+    "مليون": "1000000", "١٠٠٠٠٠٠": "1000000",
+    "ملايين": "1000000",
+    "مليار": "1000000000", "١٠٠٠٠٠٠٠٠٠": "1000000000"
+}
+
+def replace_arabic_numbers_nemo(text: str) -> str:
+    """Convert Arabic number words to digits for NeMo"""
+    for word, digit in arabic_numbers_nemo.items():
+        text = re.sub(rf"\b{word}\b", digit, text)
+    return text
+
+def convert_arabic_numbers_whisper(sentence: str) -> str:
+    """
+    Replace Arabic number words in a sentence with digits for Whisper,
+    preserving all other words and punctuation.
+    """
+    if not arabic_numbers_available or not sentence.strip():
+        return sentence
+    
+    try:
+        # Normalization step
+        replacements = {
+            "اربعة": "أربعة", "اربع": "أربع", "اثنين": "اثنان",
+            "اتنين": "اثنان", "ثلاث": "ثلاثة", "خمس": "خمسة",
+            "ست": "ستة", "سبع": "سبعة", "ثمان": "ثمانية",
+            "تسع": "تسعة", "عشر": "عشرة",
+        }
+        for wrong, correct in replacements.items():
+            sentence = re.sub(rf"\b{wrong}\b", correct, sentence)
+
+        # Split by whitespace but keep spaces
+        words = re.split(r'(\s+)', sentence)
+        converted_words = []
+
+        for word in words:
+            stripped = word.strip()
+            if not stripped:  # skip spaces
+                converted_words.append(word)
+                continue
+
+            try:
+                num = text2number(stripped)
+                if isinstance(num, int):
+                    if num != 0 or stripped == "صفر":
+                        converted_words.append(str(num))
+                    else:
+                        converted_words.append(word)
+                else:
+                    converted_words.append(word)
+            except Exception:
+                converted_words.append(word)
+
+        return ''.join(converted_words)
+
+    except Exception as e:
+        logger.warning(f"Error converting Arabic numbers: {e}")
+        return sentence
+
+# Global models
+asr_model_nemo = None
+whisper_model = None
+whisper_processor = None
+whisper_tokenizer = None
+device = None
+torch_dtype = None
+
+def initialize_models():
+    """Initialize both NeMo and Whisper models"""
+    global asr_model_nemo, whisper_model, whisper_processor, whisper_tokenizer, device, torch_dtype
+    
+    # Initialize device settings
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+    
+    logger.info(f"Using device: {device}")
+    logger.info(f"CUDA available: {torch.cuda.is_available()}")
+    
+    # Initialize NeMo model
+    logger.info("Loading NeMo FastConformer Arabic ASR model...")
+    model_path = "stt_ar_fastconformer_hybrid_large_pcd_v1.0.nemo"
+    
+    if os.path.exists(model_path):
+        try:
+            asr_model_nemo = nemo_asr.models.EncDecCTCModel.restore_from(model_path)
+            asr_model_nemo.eval()
+            logger.info("✓ NeMo FastConformer model loaded successfully")
+        except Exception as e:
+            logger.error(f"Failed to load NeMo model: {e}")
+            asr_model_nemo = None
+    else:
+        logger.warning(f"NeMo model not found at: {model_path}")
+        asr_model_nemo = None
+    
+    # Initialize Whisper model
+    # from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
+
+    logger.info("Loading Whisper large-v3 model...")
+    MODEL_NAME = "alaatiger989/FT_Arabic_Whisper_V1_1"
+
+    try:
+        # Try with flash attention first
+        try:
+            import flash_attn
+            whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+                MODEL_NAME,
+                torch_dtype=torch_dtype,
+                low_cpu_mem_usage=True,
+                use_safetensors=True,
+                attn_implementation="flash_attention_2"
+            )
+            logger.info("✓ Whisper loaded with flash attention")
+        except:
+            whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+                MODEL_NAME,
+                torch_dtype=torch_dtype,
+                low_cpu_mem_usage=True,
+                use_safetensors=True
+            )
+            logger.info("✓ Whisper loaded with standard attention")
+
+        whisper_model.to(device)
+        whisper_processor = AutoProcessor.from_pretrained(MODEL_NAME)
+
+        # Use processor.tokenizer, don’t reload separately
+        whisper_tokenizer = whisper_processor.tokenizer  
+
+        logger.info("✓ Whisper model + tokenizer loaded successfully")
+
+    except Exception as e:
+        logger.error(f"Failed to load Whisper model: {e}")
+        whisper_model = None
+
+
+
+
+    # logger.info("Loading Whisper large-v3-turbo model...")
+    # MODEL_NAME = "openai/whisper-large-v3-turbo"
+    
+    # try:
+    #     # Try with flash attention first
+    #     try:
+    #         import flash_attn
+    #         whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+    #             MODEL_NAME, 
+    #             torch_dtype=torch_dtype, 
+    #             low_cpu_mem_usage=True, 
+    #             use_safetensors=True, 
+    #             attn_implementation="flash_attention_2"
+    #         )
+    #         logger.info("✓ Whisper loaded with flash attention")
+    #     except:
+    #         whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+    #             MODEL_NAME, 
+    #             torch_dtype=torch_dtype, 
+    #             low_cpu_mem_usage=True, 
+    #             use_safetensors=True
+    #         )
+    #         logger.info("✓ Whisper loaded with standard attention")
+        
+    #     whisper_model.to(device)
+    #     whisper_processor = AutoProcessor.from_pretrained(MODEL_NAME)
+    #     whisper_tokenizer = WhisperTokenizer.from_pretrained(MODEL_NAME)
+    #     logger.info("✓ Whisper model loaded successfully")
+        
+    # except Exception as e:
+    #     logger.error(f"Failed to load Whisper model: {e}")
+    #     whisper_model = None
+
+# Initialize models on startup
+initialize_models()
+
+# Thread pool for processing
+executor = ThreadPoolExecutor(max_workers=4)
+
+# class JambonzAudioBuffer:
+#     def __init__(self, sample_rate=8000, chunk_duration=1.0):
+#         self.sample_rate = sample_rate
+#         self.chunk_duration = chunk_duration
+#         self.chunk_samples = int(chunk_duration * sample_rate)
+        
+#         self.buffer = np.array([], dtype=np.float32)
+#         self.lock = threading.Lock()
+#         self.total_audio = np.array([], dtype=np.float32)
+        
+#         # Voice Activity Detection
+#         self.silence_threshold = 0.05
+#         self.min_speech_samples = int(0.5 * sample_rate)
+        
+#     def add_audio(self, audio_data):
+#         with self.lock:
+#             self.buffer = np.concatenate([self.buffer, audio_data])
+#             self.total_audio = np.concatenate([self.total_audio, audio_data])
+    
+#     def has_chunk_ready(self):
+#         with self.lock:
+#             return len(self.buffer) >= self.chunk_samples
+    
+#     def is_speech(self, audio_chunk):
+#         """Simple VAD based on energy"""
+#         if len(audio_chunk) < self.min_speech_samples:
+#             return False
+#         energy = np.mean(np.abs(audio_chunk))
+#         return energy > self.silence_threshold
+    
+#     def get_chunk_for_processing(self):
+#         """Get audio chunk for processing"""
+#         with self.lock:
+#             if len(self.buffer) < self.chunk_samples:
+#                 return None
+#             return np.array([1])  # Signal that chunk is ready
+    
+#     def get_all_audio(self):
+#         """Get all accumulated audio"""
+#         with self.lock:
+#             return self.total_audio.copy()
+    
+#     def clear(self):
+#         with self.lock:
+#             self.buffer = np.array([], dtype=np.float32)
+#             self.total_audio = np.array([], dtype=np.float32)
+    
+#     def reset_for_new_segment(self):
+#         """Reset buffers for new transcription segment"""
+#         with self.lock:
+#             self.buffer = np.array([], dtype=np.float32)
+#             self.total_audio = np.array([], dtype=np.float32)
+
+class JambonzAudioBuffer:
+    def __init__(self, sample_rate=8000, chunk_duration=1.0):
+        self.sample_rate = sample_rate
+        self.chunk_duration = chunk_duration
+        self.chunk_samples = int(chunk_duration * sample_rate)
+        
+        self.buffer = np.array([], dtype=np.float32)
+        self.lock = threading.Lock()
+        self.total_audio = np.array([], dtype=np.float32)
+        
+        # Voice Activity Detection - ADJUSTED FOR WHISPER
+        self.silence_threshold = 0.01  # Lower threshold for Whisper
+        self.min_speech_samples = int(0.3 * sample_rate)  # 300ms minimum speech
+        
+    def add_audio(self, audio_data):
+        with self.lock:
+            self.buffer = np.concatenate([self.buffer, audio_data])
+            self.total_audio = np.concatenate([self.total_audio, audio_data])
+            
+            # Log audio addition for debugging
+            logger.debug(f"Added {len(audio_data)} audio samples, total: {len(self.total_audio)}")
+    
+    def has_chunk_ready(self):
+        with self.lock:
+            ready = len(self.buffer) >= self.chunk_samples
+            if ready:
+                logger.debug(f"Chunk ready: {len(self.buffer)} >= {self.chunk_samples}")
+            return ready
+    
+    def is_speech(self, audio_chunk):
+        """Enhanced VAD based on energy - better for Whisper"""
+        if len(audio_chunk) < self.min_speech_samples:
+            logger.debug(f"Audio too short for VAD: {len(audio_chunk)} < {self.min_speech_samples}")
+            return False
+            
+        # Calculate RMS energy
+        rms_energy = np.sqrt(np.mean(audio_chunk ** 2))
+        
+        # Also check peak amplitude
+        peak_amplitude = np.max(np.abs(audio_chunk))
+        
+        is_speech = rms_energy > self.silence_threshold or peak_amplitude > (self.silence_threshold * 2)
+        
+        logger.debug(f"VAD check - RMS: {rms_energy:.4f}, Peak: {peak_amplitude:.4f}, "
+                    f"Threshold: {self.silence_threshold}, Speech: {is_speech}")
+        
+        return is_speech
+    
+    def get_chunk_for_processing(self):
+        """Get audio chunk for processing"""
+        with self.lock:
+            if len(self.buffer) < self.chunk_samples:
+                return None
+            
+            logger.debug(f"Returning processing signal, buffer size: {len(self.buffer)}")
+            return np.array([1])  # Signal that chunk is ready
+    
+    def get_all_audio(self):
+        """Get all accumulated audio"""
+        with self.lock:
+            audio_copy = self.total_audio.copy()
+            logger.debug(f"Returning {len(audio_copy)} total audio samples")
+            return audio_copy
+    
+    def clear(self):
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+            logger.debug("Audio buffer cleared")
+    
+    def reset_for_new_segment(self):
+        """Reset buffers for new transcription segment"""
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+            logger.debug("Audio buffer reset for new segment")
+
+def linear16_to_audio(audio_bytes, sample_rate=8000):
+    """Convert LINEAR16 PCM bytes to numpy array"""
+    try:
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16)
+        audio_array = audio_array.astype(np.float32) / 32768.0
+        return audio_array
+    except Exception as e:
+        logger.error(f"Error converting LINEAR16 to audio: {e}")
+        return np.array([], dtype=np.float32)
+
+def resample_audio(audio_data, source_rate, target_rate):
+    """Resample audio to target sample rate"""
+    if source_rate == target_rate:
+        return audio_data
+    
+    if source_rate == 8000 and target_rate == 16000:
+        # Simple 2x upsampling for common case
+        upsampled = np.repeat(audio_data, 2)
+        return upsampled.astype(np.float32)
+    
+    # Fallback: Linear interpolation resampling
+    ratio = target_rate / source_rate
+    indices = np.arange(0, len(audio_data), 1/ratio)
+    indices = indices[indices < len(audio_data)]
+    resampled = np.interp(indices, np.arange(len(audio_data)), audio_data)
+    
+    return resampled.astype(np.float32)
+
+def transcribe_with_nemo(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio using NeMo FastConformer"""
+    try:
+        if len(audio_data) == 0 or asr_model_nemo is None:
+            return ""
+        
+        # Resample to 16kHz (NeMo models typically expect 16kHz)
+        resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        
+        # Skip very short audio
+        min_samples = int(0.3 * target_sample_rate)
+        if len(resampled_audio) < min_samples:
+            return ""
+        
+        start_time = time.time()
+        
+        # Save audio to temporary file (NeMo expects file path)
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
+            sf.write(tmp_file.name, resampled_audio, target_sample_rate)
+            tmp_path = tmp_file.name
+        
+        try:
+            # Transcribe with NeMo
+            result = asr_model_nemo.transcribe([tmp_path])
+            
+            if result and len(result) > 0:
+                # Handle different NeMo result formats
+                if hasattr(result[0], 'text'):
+                    raw_text = result[0].text
+                elif isinstance(result[0], str):
+                    raw_text = result[0]
+                else:
+                    raw_text = str(result[0])
+                
+                if not isinstance(raw_text, str):
+                    raw_text = str(raw_text)
+                
+                if raw_text and raw_text.strip():
+                    # Convert Arabic numbers to digits for NeMo
+                    cleaned_text = replace_arabic_numbers_nemo(raw_text)
+                    end_time = time.time()
+                    
+                    if cleaned_text.strip():
+                        logger.info(f"NeMo transcription: '{cleaned_text}' (processed in {end_time - start_time:.2f}s)")
+                    
+                    return cleaned_text.strip()
+                    
+        finally:
+            # Clean up temporary file
+            if os.path.exists(tmp_path):
+                os.remove(tmp_path)
+        
+        return ""
+        
+    except Exception as e:
+        logger.error(f"Error during NeMo transcription: {e}")
+        return ""
+
+def transcribe_with_whisper(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio chunk using Whisper model directly"""
+    try:
+        if len(audio_data) == 0 or whisper_model is None:
+            return ""
+        
+        # Resample from 8kHz to 16kHz for Whisper
+        resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        
+        # Ensure minimum length for Whisper
+        min_samples = int(0.1 * target_sample_rate)  # 100ms minimum
+        if len(resampled_audio) < min_samples:
+            return ""
+        
+        start_time = time.time()
+        
+        # Prepare input features with proper dtype
+        input_features = whisper_processor(
+            resampled_audio, 
+            sampling_rate=target_sample_rate, 
+            return_tensors="pt"
+        ).input_features
+        
+        # Ensure correct dtype and device
+        input_features = input_features.to(device=device, dtype=torch_dtype)
+        
+        # Create attention mask to avoid warnings
+        attention_mask = torch.ones(
+            input_features.shape[:-1], 
+            dtype=torch.long, 
+            device=device
+        )
+        
+        # Generate transcription using model directly
+        with torch.no_grad():
+            predicted_ids = whisper_model.generate(
+                input_features,
+                attention_mask=attention_mask,
+                max_new_tokens=128,
+                do_sample=False,
+                # temperature=0.0,
+                num_beams=1,
+                language="english",
+                task="translate",
+                pad_token_id=whisper_tokenizer.pad_token_id,
+                eos_token_id=whisper_tokenizer.eos_token_id
+            )
+        
+        # Decode the transcription
+        transcription = whisper_tokenizer.batch_decode(
+            predicted_ids, 
+            skip_special_tokens=True
+        )[0].strip()
+        
+        end_time = time.time()
+        
+        logger.info(f"Whisper transcription completed in {end_time - start_time:.2f}s: '{transcription}'")
+        return transcription
+        
+    except Exception as e:
+        logger.error(f"Error during Whisper transcription: {e}")
+        return ""
+
+class UnifiedSTTHandler:
+    def __init__(self, websocket):
+        self.websocket = websocket
+        self.audio_buffer = None
+        self.config = {}
+        self.running = False
+        self.transcription_task = None
+        self.use_nemo = False  # Flag to determine which model to use
+        
+        # Auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.silence_timeout = 2.9
+        self.min_interim_count = 1
+        self.auto_final_task = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.segment_number = 0
+        self.last_partial = ""
+        
+        # Processing tracking
+        self.processing_count = 0
+
+    # Add this debugging method to your UnifiedSTTHandler class
+
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer with enhanced debugging"""
+        if self.audio_buffer and self.running:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+            
+            model_name = "NeMo" if self.use_nemo else "Whisper"
+            
+            # Debug logging every few audio packets
+            if len(audio_data) > 0:
+                total_samples = len(self.audio_buffer.get_all_audio())
+                total_seconds = total_samples / self.config["sample_rate"]
+                
+                # Log every second of audio
+                if int(total_seconds) != getattr(self, '_last_logged_second', -1):
+                    logger.info(f"{model_name} - Accumulated {total_seconds:.1f}s of audio ({total_samples} samples)")
+                    self._last_logged_second = int(total_seconds)
+                    
+                    # Check if we should have chunks ready
+                    chunk_ready = self.audio_buffer.has_chunk_ready()
+                    logger.info(f"{model_name} - Chunk ready: {chunk_ready}")        
+    # async def start_processing(self, start_message):
+    #     """Initialize with start message from jambonz"""
+    #     self.config = {
+    #         "language": start_message.get("language", "ar-EG"),
+    #         "format": start_message.get("format", "raw"),
+    #         "encoding": start_message.get("encoding", "LINEAR16"),
+    #         "sample_rate": start_message.get("sampleRateHz", 8000),
+    #         "interim_results": True,  # Always enable for internal processing
+    #         "options": start_message.get("options", {})
+    #     }
+        
+    #     # Determine which model to use based on language parameter
+    #     language = self.config["language"]
+    #     if language == "ar-EG":
+    #         logger.info("nemooooooooooooooooooooooooooo")
+    #         self.use_nemo = True
+    #         model_name = "NeMo FastConformer"
+    #     elif language == "ar-EG-whis":
+    #         logger.info("whisperrrrrrrrrrrrrrrrrrrrrrrrrrrrr")
+    #         self.use_nemo = False
+    #         model_name = "Whisper large-v3"
+    #     else:
+    #         # Default to NeMo for any other Arabic variant
+    #         self.use_nemo = True
+    #         model_name = "NeMo FastConformer (default)"
+        
+    #     logger.info(f"STT session started with {model_name} for language: {language}")
+    #     logger.info(f"Config: {self.config}")
+        
+    #     # Check if selected model is available
+    #     if self.use_nemo and asr_model_nemo is None:
+    #         await self.send_error("NeMo model not available")
+    #         return
+    #     elif not self.use_nemo and whisper_model is None:
+    #         await self.send_error("Whisper model not available")
+    #         return
+        
+    #     # Initialize audio buffer
+    #     self.audio_buffer = JambonzAudioBuffer(
+    #         sample_rate=self.config["sample_rate"],
+    #         chunk_duration=1.0  # 1 second chunks
+    #     )
+        
+    #     # Reset session variables
+    #     self.running = True
+    #     self.interim_count = 0
+    #     self.last_interim_time = None
+    #     self.accumulated_transcript = ""
+    #     self.final_sent = False
+    #     self.segment_number = 0
+    #     self.processing_count = 0
+    #     self.last_partial = ""
+        
+    #     # Start background transcription task
+    #     self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+    #     # Start auto-final detection task
+    #     self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+    
+    # Replace these methods in your UnifiedSTTHandler class
+
+    async def start_processing(self, start_message):
+        """Initialize with start message from jambonz"""
+        self.config = {
+            "language": start_message.get("language", "ar-EG"),
+            "format": start_message.get("format", "raw"),
+            "encoding": start_message.get("encoding", "LINEAR16"),
+            "sample_rate": start_message.get("sampleRateHz", 8000),
+            "interim_results": True,  # Always enable for internal processing
+            "options": start_message.get("options", {})
+        }
+        
+        # Determine which model to use based on language parameter
+        language = self.config["language"]
+        if language == "ar-EG":
+            logger.info("Selected NeMo FastConformer")
+            self.use_nemo = True
+            model_name = "NeMo FastConformer"
+        elif language == "ar-EG-whis":
+            logger.info("Selected Whisper large-v3")
+            self.use_nemo = False
+            model_name = "Whisper large-v3"
+        else:
+            # Default to NeMo for any other Arabic variant
+            self.use_nemo = True
+            model_name = "NeMo FastConformer (default)"
+        
+        logger.info(f"STT session started with {model_name} for language: {language}")
+        logger.info(f"Config: {self.config}")
+        
+        # Check if selected model is available
+        if self.use_nemo and asr_model_nemo is None:
+            await self.send_error("NeMo model not available")
+            return
+        elif not self.use_nemo and whisper_model is None:
+            await self.send_error("Whisper model not available")
+            return
+        
+        # Initialize audio buffer with model-specific settings
+        if self.use_nemo:
+            chunk_duration = 1.0  # NeMo processes every 1 second
+        else:
+            chunk_duration = 2.0  # Whisper processes every 2 seconds for better accuracy
+        
+        self.audio_buffer = JambonzAudioBuffer(
+            sample_rate=self.config["sample_rate"],
+            chunk_duration=chunk_duration
+        )
+        
+        # Adjust VAD threshold for Whisper
+        if not self.use_nemo:
+            self.audio_buffer.silence_threshold = 0.005  # Lower threshold for Whisper
+        
+        # Reset session variables
+        self.running = True
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.segment_number = 0
+        self.processing_count = 0
+        self.last_partial = ""
+        
+        # Start background transcription task
+        self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+        # Start auto-final detection task
+        self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+        
+        logger.info(f"Background tasks started for {model_name}")
+
+
+
+    async def stop_processing(self):
+        """Stop current processing session"""
+        logger.info("Stopping STT session...")
+        self.running = False
+        
+        # Cancel background tasks
+        for task in [self.transcription_task, self.auto_final_task]:
+            if task:
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+        
+        # Send final transcription if not already sent
+        if not self.final_sent and self.accumulated_transcript.strip():
+            await self.send_transcription(self.accumulated_transcript, is_final=True)
+        
+        # Process any remaining audio for comprehensive final transcription
+        if self.audio_buffer:
+            all_audio = self.audio_buffer.get_all_audio()
+            if len(all_audio) > 0 and not self.final_sent:
+                loop = asyncio.get_event_loop()
+                
+                if self.use_nemo:
+                    final_transcription = await loop.run_in_executor(
+                        executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+                    )
+                else:
+                    final_transcription = await loop.run_in_executor(
+                        executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+                    )
+                
+                if final_transcription.strip():
+                    await self.send_transcription(final_transcription, is_final=True)
+        
+        # Clear audio buffer
+        if self.audio_buffer:
+            self.audio_buffer.clear()
+        
+        logger.info("STT session stopped")
+    
+    async def start_new_segment(self):
+        """Start a new transcription segment"""
+        self.segment_number += 1
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.last_partial = ""
+        self.processing_count = 0
+        
+        if self.audio_buffer:
+            self.audio_buffer.reset_for_new_segment()
+        
+        logger.info(f"Started new transcription segment #{self.segment_number}")
+    
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer"""
+        if self.audio_buffer and self.running:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+    
+    # async def _process_audio_chunks(self):
+    #     """Process audio chunks for interim results"""
+    #     while self.running:
+    #         try:
+    #             if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+    #                 chunk_signal = self.audio_buffer.get_chunk_for_processing()
+    #                 if chunk_signal is not None:
+    #                     all_audio = self.audio_buffer.get_all_audio()
+                        
+    #                     if len(all_audio) > 0 and self.audio_buffer.is_speech(all_audio[-self.audio_buffer.chunk_samples:]):
+    #                         loop = asyncio.get_event_loop()
+                            
+    #                         # Choose transcription method based on model selection
+    #                         if self.use_nemo:
+    #                             transcription = await loop.run_in_executor(
+    #                                 executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+    #                             )
+    #                         else:
+    #                             transcription = await loop.run_in_executor(
+    #                                 executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+    #                             )
+                            
+    #                         if transcription.strip():
+    #                             self.processing_count += 1
+    #                             self.accumulated_transcript = transcription
+                                
+    #                             if transcription != self.last_partial or self.interim_count == 0:
+    #                                 self.last_partial = transcription
+    #                                 self.interim_count += 1
+    #                                 self.last_interim_time = time.time()
+    #                                 logger.info(f"Updated interim_count to {self.interim_count} for transcript: '{transcription}'")
+    #                             else:
+    #                                 self.last_interim_time = time.time()
+                
+    #             await asyncio.sleep(0.1)  # Check every 100ms
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in chunk processing: {e}")
+    #             await asyncio.sleep(0.1)
+    
+    
+    # async def _monitor_for_auto_final(self):
+    #     """Monitor for auto-final conditions"""
+    #     while self.running:
+    #         try:
+    #             current_time = time.time()
+                
+    #             if (self.interim_count >= self.min_interim_count and 
+    #                 self.last_interim_time is not None and 
+    #                 (current_time - self.last_interim_time) >= self.silence_timeout and
+    #                 not self.final_sent and
+    #                 self.accumulated_transcript.strip()):
+                    
+    #                 logger.info(f"Auto-final triggered for segment #{self.segment_number}")
+                    
+    #                 await self.send_transcription(self.accumulated_transcript, is_final=True)
+    #                 await self.start_new_segment()
+                
+    #             await asyncio.sleep(0.5)  # Check every 500ms
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in auto-final monitoring: {e}")
+    #             await asyncio.sleep(0.5)
+    
+    # async def _process_audio_chunks(self):
+    #     """Process audio chunks for interim results - FIXED for Whisper streaming"""
+    #     logger.info(f"Starting audio chunk processing for {'NeMo' if self.use_nemo else 'Whisper'}")
+        
+    #     while self.running:
+    #         try:
+    #             if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+    #                 chunk_signal = self.audio_buffer.get_chunk_for_processing()
+    #                 if chunk_signal is not None:
+    #                     all_audio = self.audio_buffer.get_all_audio()
+                        
+    #                     # Check if we have enough audio and speech activity
+    #                     if len(all_audio) > 0:
+    #                         # Get the latest chunk for VAD check
+    #                         latest_chunk_start = max(0, len(all_audio) - self.audio_buffer.chunk_samples)
+    #                         latest_chunk = all_audio[latest_chunk_start:]
+                            
+    #                         # For debugging
+    #                         logger.debug(f"Audio buffer size: {len(all_audio)} samples, Latest chunk: {len(latest_chunk)} samples")
+                            
+    #                         if self.audio_buffer.is_speech(latest_chunk):
+    #                             logger.info(f"Speech detected, processing with {'NeMo' if self.use_nemo else 'Whisper'}")
+                                
+    #                             loop = asyncio.get_event_loop()
+                                
+    #                             # Choose transcription method based on model selection
+    #                             if self.use_nemo:
+    #                                 transcription = await loop.run_in_executor(
+    #                                     executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+    #                                 )
+    #                             else:
+    #                                 # For Whisper, ensure we process the accumulated audio
+    #                                 transcription = await loop.run_in_executor(
+    #                                     executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+    #                                 )
+                                
+    #                             logger.info(f"Transcription result: '{transcription}'")
+                                
+    #                             if transcription.strip():
+    #                                 self.processing_count += 1
+    #                                 self.accumulated_transcript = transcription
+                                    
+    #                                 if transcription != self.last_partial or self.interim_count == 0:
+    #                                     self.last_partial = transcription
+    #                                     self.interim_count += 1
+    #                                     self.last_interim_time = time.time()
+    #                                     logger.info(f"Updated interim_count to {self.interim_count} for transcript: '{transcription}'")
+    #                                 else:
+    #                                     self.last_interim_time = time.time()
+    #                                     logger.info("Same transcription, updating time only")
+    #                         else:
+    #                             logger.debug("No speech detected in latest chunk")
+                
+    #             await asyncio.sleep(0.1)  # Check every 100ms
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in chunk processing: {e}")
+    #             import traceback
+    #             traceback.print_exc()
+    #             await asyncio.sleep(0.1)
+    
+    # async def _monitor_for_auto_final(self):
+    #     """Monitor for auto-final conditions - Enhanced logging"""
+    #     logger.info("Starting auto-final monitoring")
+        
+    #     while self.running:
+    #         try:
+    #             current_time = time.time()
+                
+    #             if (self.interim_count >= self.min_interim_count and 
+    #                 self.last_interim_time is not None and 
+    #                 (current_time - self.last_interim_time) >= self.silence_timeout and
+    #                 not self.final_sent and
+    #                 self.accumulated_transcript.strip()):
+                    
+    #                 silence_duration = current_time - self.last_interim_time
+    #                 logger.info(f"Auto-final triggered for segment #{self.segment_number} - "
+    #                           f"Interim count: {self.interim_count}, Silence: {silence_duration:.1f}s")
+                    
+    #                 await self.send_transcription(self.accumulated_transcript, is_final=True)
+    #                 await self.start_new_segment()
+                
+    #             # Debug logging every 5 seconds
+    #             if int(current_time) % 5 == 0:
+    #                 logger.debug(f"Auto-final status - Interim count: {self.interim_count}, "
+    #                            f"Last interim: {self.last_interim_time}, "
+    #                            f"Final sent: {self.final_sent}, "
+    #                            f"Transcript: '{self.accumulated_transcript[:50]}...'")
+                
+    #             await asyncio.sleep(0.5)  # Check every 500ms
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in auto-final monitoring: {e}")
+    #             await asyncio.sleep(0.5)
+    
+    # async def _process_audio_chunks(self):
+    #     """Process audio chunks for interim results - FIXED for both models"""
+    #     model_name = "NeMo" if self.use_nemo else "Whisper"
+    #     logger.info(f"Starting audio chunk processing for {model_name}")
+        
+    #     while self.running:
+    #         try:
+    #             if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+    #                 chunk_signal = self.audio_buffer.get_chunk_for_processing()
+    #                 if chunk_signal is not None:
+    #                     all_audio = self.audio_buffer.get_all_audio()
+                        
+    #                     # Debug logging
+    #                     logger.debug(f"Processing chunk - Total audio: {len(all_audio)} samples")
+                        
+    #                     if len(all_audio) > 0:
+    #                         # Get the latest chunk for VAD check
+    #                         latest_chunk_start = max(0, len(all_audio) - self.audio_buffer.chunk_samples)
+    #                         latest_chunk = all_audio[latest_chunk_start:]
+                            
+    #                         # Check for speech activity
+    #                         has_speech = self.audio_buffer.is_speech(latest_chunk)
+    #                         logger.debug(f"Speech detection result: {has_speech}")
+                            
+    #                         if has_speech:
+    #                             logger.info(f"Processing audio with {model_name} - {len(all_audio)} samples")
+                                
+    #                             loop = asyncio.get_event_loop()
+    #                             start_time = time.time()
+                                
+    #                             try:
+    #                                 # Choose transcription method based on model selection
+    #                                 if self.use_nemo:
+    #                                     transcription = await loop.run_in_executor(
+    #                                         executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+    #                                     )
+    #                                 else:
+    #                                     # For Whisper, ensure we have enough audio
+    #                                     if len(all_audio) >= int(0.5 * 16000):  # At least 0.5 seconds at 16kHz
+    #                                         transcription = await loop.run_in_executor(
+    #                                             executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+    #                                         )
+    #                                     else:
+    #                                         transcription = ""
+    #                                         logger.debug("Whisper: Not enough audio for transcription")
+                                    
+    #                                 process_time = time.time() - start_time
+    #                                 logger.info(f"{model_name} processing took {process_time:.2f}s, result: '{transcription}'")
+                                    
+    #                                 if transcription and transcription.strip():
+    #                                     self.processing_count += 1
+    #                                     self.accumulated_transcript = transcription
+                                        
+    #                                     if transcription != self.last_partial or self.interim_count == 0:
+    #                                         self.last_partial = transcription
+    #                                         self.interim_count += 1
+    #                                         self.last_interim_time = time.time()
+    #                                         logger.info(f"Updated interim_count to {self.interim_count} for transcript: '{transcription}'")
+    #                                     else:
+    #                                         self.last_interim_time = time.time()
+    #                                         logger.debug("Same transcription, updating time only")
+    #                                 else:
+    #                                     logger.debug(f"{model_name} returned empty transcription")
+                                        
+    #                             except Exception as e:
+    #                                 logger.error(f"Error in {model_name} transcription: {e}")
+    #                         else:
+    #                             logger.debug("No speech detected in latest chunk")
+                
+    #             # Different sleep intervals for different models
+    #             sleep_interval = 0.1 if self.use_nemo else 0.2  # Whisper can be less frequent
+    #             await asyncio.sleep(sleep_interval)
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in chunk processing: {e}")
+    #             import traceback
+    #             traceback.print_exc()
+    #             await asyncio.sleep(1)  # Longer sleep on error
+
+    # Also add this to the beginning of _process_audio_chunks method:
+    
+    async def _process_audio_chunks(self):
+        """Process audio chunks for interim results - with debugging"""
+        model_name = "NeMo" if self.use_nemo else "Whisper"
+        logger.info(f"Starting audio chunk processing for {model_name}")
+        
+        chunk_count = 0
+        
+        while self.running:
+            try:
+                if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+                    chunk_count += 1
+                    logger.info(f"{model_name} - Processing chunk #{chunk_count}")
+                    
+                    chunk_signal = self.audio_buffer.get_chunk_for_processing()
+                    if chunk_signal is not None:
+                        all_audio = self.audio_buffer.get_all_audio()
+                        
+                        logger.info(f"{model_name} - Got {len(all_audio)} samples for processing")
+                        
+                        if len(all_audio) > 0:
+                            # Get the latest chunk for VAD check
+                            latest_chunk_start = max(0, len(all_audio) - self.audio_buffer.chunk_samples)
+                            latest_chunk = all_audio[latest_chunk_start:]
+                            
+                            # Check for speech activity
+                            has_speech = self.audio_buffer.is_speech(latest_chunk)
+                            logger.info(f"{model_name} - Speech detected: {has_speech}")
+                            
+                            if has_speech:
+                                logger.info(f"{model_name} - Starting transcription...")
+                                
+                                loop = asyncio.get_event_loop()
+                                start_time = time.time()
+                                
+                                try:
+                                    # Choose transcription method based on model selection
+                                    if self.use_nemo:
+                                        transcription = await loop.run_in_executor(
+                                            executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+                                        )
+                                    else:
+                                        transcription = await loop.run_in_executor(
+                                            executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+                                        )
+                                    
+                                    process_time = time.time() - start_time
+                                    logger.info(f"{model_name} - Transcription completed in {process_time:.2f}s: '{transcription}'")
+                                    
+                                    if transcription and transcription.strip():
+                                        self.processing_count += 1
+                                        self.accumulated_transcript = transcription
+                                        
+                                        if transcription != self.last_partial or self.interim_count == 0:
+                                            self.last_partial = transcription
+                                            self.interim_count += 1
+                                            self.last_interim_time = time.time()
+                                            logger.info(f"{model_name} - Updated interim_count to {self.interim_count}")
+                                        else:
+                                            self.last_interim_time = time.time()
+                                            logger.info(f"{model_name} - Same transcription, updating time only")
+                                    else:
+                                        logger.info(f"{model_name} - No transcription result")
+                                        
+                                except Exception as e:
+                                    logger.error(f"{model_name} - Transcription error: {e}")
+                                    import traceback
+                                    traceback.print_exc()
+                            else:
+                                logger.debug(f"{model_name} - No speech in chunk")
+                    else:
+                        logger.warning(f"{model_name} - Chunk signal was None")
+                else:
+                    # Log why chunk is not ready
+                    if self.audio_buffer:
+                        current_size = len(self.audio_buffer.buffer)
+                        required_size = self.audio_buffer.chunk_samples
+                        if current_size > 0:
+                            logger.debug(f"{model_name} - Buffer: {current_size}/{required_size} samples")
+                
+                await asyncio.sleep(0.1)
+                
+            except Exception as e:
+                logger.error(f"{model_name} - Error in chunk processing: {e}")
+                import traceback
+                traceback.print_exc()
+                await asyncio.sleep(1)
+
+    async def _monitor_for_auto_final(self):
+        """Monitor for auto-final conditions with model-specific timeouts"""
+        model_name = "NeMo" if self.use_nemo else "Whisper"
+        timeout = 2.0 if self.use_nemo else 3.0  # Longer timeout for Whisper
+        
+        logger.info(f"Starting auto-final monitoring for {model_name} (timeout: {timeout}s)")
+        
+        while self.running:
+            try:
+                current_time = time.time()
+                
+                if (self.interim_count >= self.min_interim_count and 
+                    self.last_interim_time is not None and 
+                    (current_time - self.last_interim_time) >= timeout and
+                    not self.final_sent and
+                    self.accumulated_transcript.strip()):
+                    
+                    silence_duration = current_time - self.last_interim_time
+                    logger.info(f"Auto-final triggered for segment #{self.segment_number} ({model_name}) - "
+                            f"Interim count: {self.interim_count}, Silence: {silence_duration:.1f}s")
+                    
+                    await self.send_transcription(self.accumulated_transcript, is_final=True)
+                    await self.start_new_segment()
+                
+                await asyncio.sleep(0.5)  # Check every 500ms
+                
+            except Exception as e:
+                logger.error(f"Error in auto-final monitoring: {e}")
+                await asyncio.sleep(0.5)
+
+
+
+    async def send_transcription(self, text, is_final=True, confidence=0.9):
+        """Send transcription in jambonz format"""
+        try:
+            # Apply number conversion only for Whisper
+            if not self.use_nemo and is_final:
+                original_text = text
+                converted_text = convert_arabic_numbers_whisper(text)
+                
+                if original_text != converted_text:
+                    logger.info(f"Whisper - Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+                text = converted_text
+            
+            message = {
+                "type": "transcription",
+                "is_final": True,  # Always send as final
+                "alternatives": [
+                    {
+                        "transcript": text,
+                        "confidence": confidence
+                    }
+                ],
+                "language": self.config.get("language", "ar-EG"),
+                "channel": 1
+            }
+            
+            await self.websocket.send(json.dumps(message))
+            self.final_sent = True
+            
+            model_name = "NeMo" if self.use_nemo else "Whisper"
+            logger.info(f"Sent FINAL transcription ({model_name}): '{text}'")
+            
+        except Exception as e:
+            logger.error(f"Error sending transcription: {e}")
+    
+    async def send_error(self, error_message):
+        """Send error message in jambonz format"""
+        try:
+            message = {
+                "type": "error",
+                "error": error_message
+            }
+            await self.websocket.send(json.dumps(message))
+            logger.error(f"Sent error: {error_message}")
+        except Exception as e:
+            logger.error(f"Error sending error message: {e}")
+
+async def handle_jambonz_websocket(websocket):
+    """Handle jambonz WebSocket connections"""
+    
+    client_id = f"jambonz_{id(websocket)}"
+    logger.info(f"New unified STT connection: {client_id}")
+    
+    handler = UnifiedSTTHandler(websocket)
+    
+    try:
+        async for message in websocket:
+            try:
+                if isinstance(message, str):
+                    data = json.loads(message)
+                    message_type = data.get("type")
+                    
+                    if message_type == "start":
+                        logger.info(f"Received start message: {data}")
+                        await handler.start_processing(data)
+                        
+                    elif message_type == "stop":
+                        logger.info("Received stop message - closing WebSocket")
+                        await handler.stop_processing()
+                        await websocket.close(code=1000, reason="Session stopped by client")
+                        break
+                        
+                    else:
+                        logger.warning(f"Unknown message type: {message_type}")
+                        await handler.send_error(f"Unknown message type: {message_type}")
+                
+                else:
+                    # Handle binary audio data
+                    if not handler.running or handler.audio_buffer is None:
+                        logger.warning("Received audio data outside of active session")
+                        await handler.send_error("Received audio before start message or after stop")
+                        continue
+                    
+                    await handler.add_audio_data(message)
+            
+            except json.JSONDecodeError as e:
+                logger.error(f"JSON decode error: {e}")
+                await handler.send_error(f"Invalid JSON: {str(e)}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+                await handler.send_error(f"Processing error: {str(e)}")
+    
+    except websockets.exceptions.ConnectionClosed:
+        logger.info(f"Unified STT connection closed: {client_id}")
+    except Exception as e:
+        logger.error(f"Unified STT WebSocket error: {e}")
+        try:
+            await handler.send_error(str(e))
+        except:
+            pass
+    finally:
+        if handler.running:
+            await handler.stop_processing()
+        logger.info(f"Unified STT connection ended: {client_id}")
+
+async def main():
+    """Start the Unified Arabic STT WebSocket server"""
+    logger.info("Starting Unified Arabic STT WebSocket server on port 3007...")
+    
+    # Check model availability
+    models_available = []
+    if asr_model_nemo is not None:
+        models_available.append("NeMo FastConformer (ar-EG)")
+    if whisper_model is not None:
+        models_available.append("Whisper large-v3 (ar-EG-whis)")
+    
+    if not models_available:
+        logger.error("No models available! Please check model paths and installations.")
+        return
+    
+    # Start WebSocket server
+    server = await websockets.serve(
+        handle_jambonz_websocket,
+        "0.0.0.0", 
+        3007,
+        ping_interval=20,
+        ping_timeout=10,
+        close_timeout=10
+    )
+    
+    logger.info("Unified Arabic STT WebSocket server started on ws://0.0.0.0:3007")
+    logger.info("Ready to handle jambonz STT requests with both models")
+    logger.info("ROUTING:")
+    logger.info("- language: 'ar-EG' → NeMo FastConformer (with built-in number conversion)")
+    logger.info("- language: 'ar-EG-whis' → Whisper large-v3 (with pyarabic number conversion)")
+    logger.info("FEATURES:")
+    logger.info("- Continuous transcription with segmentation")
+    logger.info("- Voice Activity Detection")
+    logger.info("- Auto-final detection (2s silence timeout)")
+    logger.info("- Model-specific number conversion")
+    logger.info(f"AVAILABLE MODELS: {', '.join(models_available)}")
+    
+    # Wait for the server to close
+    await server.wait_closed()
+
+if __name__ == "__main__":
+    print("=" * 80)
+    print("Unified Arabic STT Server (NeMo + Whisper)")
+    print("=" * 80)
+    print("WebSocket Port: 3007")
+    print("Protocol: jambonz STT API")
+    print("Audio Format: LINEAR16 PCM @ 8kHz → 16kHz")
+    print()
+    print("LANGUAGE ROUTING:")
+    print("- 'ar-EG' → NeMo FastConformer")
+    print("  • Built-in Arabic number word to digit conversion")
+    print("  • Optimized for Arabic dialects")
+    print("- 'ar-EG-whis' → Whisper large-v3")
+    print("  • pyarabic library number conversion (final transcripts only)")
+    print("  • OpenAI Whisper model")
+    print()
+    print("FEATURES:")
+    print("- Automatic model selection based on language parameter")
+    print("- Voice Activity Detection")
+    print("- Auto-final detection (2 seconds silence)")
+    print("- Model-specific number conversion strategies")
+    print("- Continuous transcription with segmentation")
+    print()
+    
+    # Check model availability for startup info
+    nemo_status = "✓ Available" if asr_model_nemo is not None else "✗ Not Available"
+    whisper_status = "✓ Available" if whisper_model is not None else "✗ Not Available"
+    arabic_numbers_status = "✓ Available" if arabic_numbers_available else "✗ Not Available (install pyarabic)"
+    
+    print("MODEL STATUS:")
+    print(f"- NeMo FastConformer: {nemo_status}")
+    print(f"- Whisper large-v3: {whisper_status}")
+    print(f"- pyarabic (Whisper numbers): {arabic_numbers_status}")
+    print("=" * 80)
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nShutting down unified server...")
+    except Exception as e:
+        print(f"Server error: {e}")

best_nemo_whisper_jambonz_denoiser.py

@@ -0,0 +1,1357 @@
+import torch
+import asyncio
+import websockets
+import json
+import threading
+import numpy as np
+import logging
+import time
+import tempfile
+import os
+import re
+from concurrent.futures import ThreadPoolExecutor
+import subprocess
+import struct
+
+# NeMo imports
+import nemo.collections.asr as nemo_asr
+import soundfile as sf
+
+# Whisper imports
+# from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, WhisperTokenizer, pipeline
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
+
+
+# Arabic number conversion imports for Whisper
+try:
+    from pyarabic.number import text2number
+    arabic_numbers_available = True
+    print("✓ pyarabic library available for Whisper number conversion")
+except ImportError:
+    arabic_numbers_available = False
+    print("✗ pyarabic not available - install with: pip install pyarabic")
+    print("Arabic numbers will not be converted to digits for Whisper")
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# ===== NeMo Arabic number mapping =====
+arabic_numbers_nemo = {
+    # Basic digits
+    "سفر": "0", "فيرو": "0", "هيرو": "0","صفر": "0", "زيرو": "0", "٠": "0","زيو": "0","زير": "0","زير": "0","زر": "0","زروا": "0","زرا": "0","زيره ": "0","زرو ": "0",
+    "واحد": "1", "واحدة": "1", "١": "1",
+    "اتنين": "2", "اثنين": "2", "إثنين": "2", "اثنان": "2", "إثنان": "2", "٢": "2",
+    "تلاتة": "3", "ثلاثة": "3", "٣": "3","تلاته": "3","ثلاثه": "3","ثلاثا": "3","تلاتا": "3",
+    "اربعة": "4", "أربعة": "4", "٤": "4","اربعه": "4","أربعه": "4","أربع": "4","اربع": "4","اربعا": "4","أربعا": "4",
+    "خمسة": "5", "خمسه": "5", "٥": "5", "خمس": "5", "خمسا": "5",
+    "ستة": "6", "سته": "6", "٦": "6", "ست": "6", "ستّا": "6", "ستةً": "6",
+    "سبعة": "7", "سبعه": "7", "٧": "7", "سبع": "7", "سبعا": "7",
+    "ثمانية": "8", "ثمانيه": "8", "٨": "8", "ثمان": "8", "ثمنية": "8", "ثمنيه": "8", "ثمانيا": "8", "ثمن": "8",
+    "تسعة": "9", "تسعه": "9", "٩": "9", "تسع": "9", "تسعا": "9",
+    
+    # Teens
+    "عشرة": "10", "١٠": "10",
+    "حداشر": "11", "احد عشر": "11","احداشر": "11",
+    "اتناشر": "12", "اثنا عشر": "12",
+    "تلتاشر": "13", "ثلاثة عشر": "13",
+    "اربعتاشر": "14", "أربعة عشر": "14",
+    "خمستاشر": "15", "خمسة عشر": "15",
+    "ستاشر": "16", "ستة عشر": "16",
+    "سبعتاشر": "17", "سبعة عشر": "17",
+    "طمنتاشر": "18", "ثمانية عشر": "18",
+    "تسعتاشر": "19", "تسعة عشر": "19",
+    
+    # Tens
+    "عشرين": "20", "٢٠": "20",
+    "تلاتين": "30", "ثلاثين": "30", "٣٠": "30",
+    "اربعين": "40", "أربعين": "40", "٤٠": "40",
+    "خمسين": "50", "٥٠": "50",
+    "ستين": "60", "٦٠": "60",
+    "سبعين": "70", "٧٠": "70",
+    "تمانين": "80", "ثمانين": "80", "٨٠": "80","تمانون": "80","ثمانون": "80",
+    "تسعين": "90", "٩٠": "90",
+    
+    # Hundreds
+    "مية": "100", "مائة": "100", "مئة": "100", "١٠٠": "100",
+    "ميتين": "200", "مائتين": "200",
+    "تلاتمية": "300", "ثلاثمائة": "300",
+    "اربعمية": "400", "أربعمائة": "400",
+    "خمسمية": "500", "خمسمائة": "500",
+    "ستمية": "600", "ستمائة": "600",
+    "سبعمية": "700", "سبعمائة": "700",
+    "تمانمية": "800", "ثمانمائة": "800",
+    "تسعمية": "900", "تسعمائة": "900",
+    
+    # Thousands
+    "ألف": "1000", "الف": "1000", "١٠٠٠": "1000",
+    "ألفين": "2000", "الفين": "2000",
+    "تلات تلاف": "3000", "ثلاثة آلاف": "3000",
+    "اربعة آلاف": "4000", "أربعة آلاف": "4000",
+    "خمسة آلاف": "5000",
+    "ستة آلاف": "6000",
+    "سبعة آلاف": "7000",
+    "تمانية آلاف": "8000", "ثمانية آلاف": "8000",
+    "تسعة آلاف": "9000",
+    
+    # Large numbers
+    "عشرة آلاف": "10000",
+    "مية ألف": "100000", "مائة ألف": "100000",
+    "مليون": "1000000", "١٠٠٠٠٠٠": "1000000",
+    "ملايين": "1000000",
+    "مليار": "1000000000", "١٠٠٠٠٠٠٠٠٠": "1000000000"
+}
+
+def replace_arabic_numbers_nemo(text: str) -> str:
+    """Convert Arabic number words to digits for NeMo"""
+    for word, digit in arabic_numbers_nemo.items():
+        text = re.sub(rf"\b{word}\b", digit, text)
+    return text
+
+def convert_arabic_numbers_whisper(sentence: str) -> str:
+    """
+    Replace Arabic number words in a sentence with digits for Whisper,
+    preserving all other words and punctuation.
+    """
+    if not arabic_numbers_available or not sentence.strip():
+        return sentence
+    
+    try:
+        # Normalization step
+        replacements = {
+            "اربعة": "أربعة", "اربع": "أربع", "اثنين": "اثنان",
+            "اتنين": "اثنان", "ثلاث": "ثلاثة", "خمس": "خمسة",
+            "ست": "ستة", "سبع": "سبعة", "ثمان": "ثمانية",
+            "تسع": "تسعة", "عشر": "عشرة",
+        }
+        for wrong, correct in replacements.items():
+            sentence = re.sub(rf"\b{wrong}\b", correct, sentence)
+
+        # Split by whitespace but keep spaces
+        words = re.split(r'(\s+)', sentence)
+        converted_words = []
+
+        for word in words:
+            stripped = word.strip()
+            if not stripped:  # skip spaces
+                converted_words.append(word)
+                continue
+
+            try:
+                num = text2number(stripped)
+                if isinstance(num, int):
+                    if num != 0 or stripped == "صفر":
+                        converted_words.append(str(num))
+                    else:
+                        converted_words.append(word)
+                else:
+                    converted_words.append(word)
+            except Exception:
+                converted_words.append(word)
+
+        return ''.join(converted_words)
+
+    except Exception as e:
+        logger.warning(f"Error converting Arabic numbers: {e}")
+        return sentence
+
+# Global models
+asr_model_nemo = None
+whisper_model = None
+whisper_processor = None
+whisper_tokenizer = None
+device = None
+torch_dtype = None
+import torch
+from denoiser import pretrained
+
+
+def initialize_models():
+    """Initialize both NeMo and Whisper models"""
+    global asr_model_nemo, whisper_model, whisper_processor, whisper_tokenizer, device, torch_dtype, denoiser_model
+    
+    # Initialize device settings
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    # Load DNS64 pretrained model (auto-downloads if not cached)
+    denoiser_model = pretrained.dns64().to(device)
+    denoiser_model.eval()
+    logger.info(f"Using device: {device}")
+    logger.info(f"CUDA available: {torch.cuda.is_available()}")
+    
+    # Initialize NeMo model
+    logger.info("Loading NeMo FastConformer Arabic ASR model...")
+    model_path = "stt_ar_fastconformer_hybrid_large_pcd_v1.0.nemo"
+    
+    if os.path.exists(model_path):
+        try:
+            asr_model_nemo = nemo_asr.models.EncDecCTCModel.restore_from(model_path)
+            asr_model_nemo.eval()
+            logger.info("✓ NeMo FastConformer model loaded successfully")
+        except Exception as e:
+            logger.error(f"Failed to load NeMo model: {e}")
+            asr_model_nemo = None
+    else:
+        logger.warning(f"NeMo model not found at: {model_path}")
+        asr_model_nemo = None
+    
+    # Initialize Whisper model
+    # from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
+
+    logger.info("Loading Whisper large-v3 model...")
+    MODEL_NAME = "alaatiger989/FT_Arabic_Whisper_V1_1"
+
+    try:
+        # Try with flash attention first
+        try:
+            import flash_attn
+            whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+                MODEL_NAME,
+                torch_dtype=torch_dtype,
+                low_cpu_mem_usage=True,
+                use_safetensors=True,
+                attn_implementation="flash_attention_2"
+            )
+            logger.info("✓ Whisper loaded with flash attention")
+        except:
+            whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+                MODEL_NAME,
+                torch_dtype=torch_dtype,
+                low_cpu_mem_usage=True,
+                use_safetensors=True
+            )
+            logger.info("✓ Whisper loaded with standard attention")
+
+        whisper_model.to(device)
+        whisper_processor = AutoProcessor.from_pretrained(MODEL_NAME)
+
+        # Use processor.tokenizer, don’t reload separately
+        whisper_tokenizer = whisper_processor.tokenizer  
+
+        logger.info("✓ Whisper model + tokenizer loaded successfully")
+
+    except Exception as e:
+        logger.error(f"Failed to load Whisper model: {e}")
+        whisper_model = None
+
+
+
+
+    # logger.info("Loading Whisper large-v3-turbo model...")
+    # MODEL_NAME = "openai/whisper-large-v3-turbo"
+    
+    # try:
+    #     # Try with flash attention first
+    #     try:
+    #         import flash_attn
+    #         whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+    #             MODEL_NAME, 
+    #             torch_dtype=torch_dtype, 
+    #             low_cpu_mem_usage=True, 
+    #             use_safetensors=True, 
+    #             attn_implementation="flash_attention_2"
+    #         )
+    #         logger.info("✓ Whisper loaded with flash attention")
+    #     except:
+    #         whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+    #             MODEL_NAME, 
+    #             torch_dtype=torch_dtype, 
+    #             low_cpu_mem_usage=True, 
+    #             use_safetensors=True
+    #         )
+    #         logger.info("✓ Whisper loaded with standard attention")
+        
+    #     whisper_model.to(device)
+    #     whisper_processor = AutoProcessor.from_pretrained(MODEL_NAME)
+    #     whisper_tokenizer = WhisperTokenizer.from_pretrained(MODEL_NAME)
+    #     logger.info("✓ Whisper model loaded successfully")
+        
+    # except Exception as e:
+    #     logger.error(f"Failed to load Whisper model: {e}")
+    #     whisper_model = None
+
+# Initialize models on startup
+initialize_models()
+def denoise_audio(audio_data, sample_rate=16000):
+    """Apply denoising using facebook/denoiser pretrained model."""
+    if denoiser_model is None or len(audio_data) == 0:
+        return audio_data
+    try:
+        audio_tensor = torch.tensor(audio_data, dtype=torch.float32, device=device).unsqueeze(0)
+        with torch.no_grad():
+            denoised_tensor = denoiser_model(audio_tensor, sample_rate=sample_rate)[0]
+        return denoised_tensor.squeeze().cpu().numpy().astype("float32")
+    except Exception as e:
+        print(f"[WARN] Denoiser failed: {e}")
+        return audio_data
+# Thread pool for processing
+executor = ThreadPoolExecutor(max_workers=4)
+
+# class JambonzAudioBuffer:
+#     def __init__(self, sample_rate=8000, chunk_duration=1.0):
+#         self.sample_rate = sample_rate
+#         self.chunk_duration = chunk_duration
+#         self.chunk_samples = int(chunk_duration * sample_rate)
+        
+#         self.buffer = np.array([], dtype=np.float32)
+#         self.lock = threading.Lock()
+#         self.total_audio = np.array([], dtype=np.float32)
+        
+#         # Voice Activity Detection
+#         self.silence_threshold = 0.05
+#         self.min_speech_samples = int(0.5 * sample_rate)
+        
+#     def add_audio(self, audio_data):
+#         with self.lock:
+#             self.buffer = np.concatenate([self.buffer, audio_data])
+#             self.total_audio = np.concatenate([self.total_audio, audio_data])
+    
+#     def has_chunk_ready(self):
+#         with self.lock:
+#             return len(self.buffer) >= self.chunk_samples
+    
+#     def is_speech(self, audio_chunk):
+#         """Simple VAD based on energy"""
+#         if len(audio_chunk) < self.min_speech_samples:
+#             return False
+#         energy = np.mean(np.abs(audio_chunk))
+#         return energy > self.silence_threshold
+    
+#     def get_chunk_for_processing(self):
+#         """Get audio chunk for processing"""
+#         with self.lock:
+#             if len(self.buffer) < self.chunk_samples:
+#                 return None
+#             return np.array([1])  # Signal that chunk is ready
+    
+#     def get_all_audio(self):
+#         """Get all accumulated audio"""
+#         with self.lock:
+#             return self.total_audio.copy()
+    
+#     def clear(self):
+#         with self.lock:
+#             self.buffer = np.array([], dtype=np.float32)
+#             self.total_audio = np.array([], dtype=np.float32)
+    
+#     def reset_for_new_segment(self):
+#         """Reset buffers for new transcription segment"""
+#         with self.lock:
+#             self.buffer = np.array([], dtype=np.float32)
+#             self.total_audio = np.array([], dtype=np.float32)
+
+class JambonzAudioBuffer:
+    def __init__(self, sample_rate=8000, chunk_duration=1.0):
+        self.sample_rate = sample_rate
+        self.chunk_duration = chunk_duration
+        self.chunk_samples = int(chunk_duration * sample_rate)
+        
+        self.buffer = np.array([], dtype=np.float32)
+        self.lock = threading.Lock()
+        self.total_audio = np.array([], dtype=np.float32)
+        
+        # Voice Activity Detection - ADJUSTED FOR WHISPER
+        self.silence_threshold = 0.01  # Lower threshold for Whisper
+        self.min_speech_samples = int(0.3 * sample_rate)  # 300ms minimum speech
+        
+    def add_audio(self, audio_data):
+        with self.lock:
+            self.buffer = np.concatenate([self.buffer, audio_data])
+            self.total_audio = np.concatenate([self.total_audio, audio_data])
+            
+            # Log audio addition for debugging
+            logger.debug(f"Added {len(audio_data)} audio samples, total: {len(self.total_audio)}")
+    
+    def has_chunk_ready(self):
+        with self.lock:
+            ready = len(self.buffer) >= self.chunk_samples
+            if ready:
+                logger.debug(f"Chunk ready: {len(self.buffer)} >= {self.chunk_samples}")
+            return ready
+    
+    def is_speech(self, audio_chunk):
+        """Enhanced VAD based on energy - better for Whisper"""
+        if len(audio_chunk) < self.min_speech_samples:
+            logger.debug(f"Audio too short for VAD: {len(audio_chunk)} < {self.min_speech_samples}")
+            return False
+            
+        # Calculate RMS energy
+        rms_energy = np.sqrt(np.mean(audio_chunk ** 2))
+        
+        # Also check peak amplitude
+        peak_amplitude = np.max(np.abs(audio_chunk))
+        
+        is_speech = rms_energy > self.silence_threshold or peak_amplitude > (self.silence_threshold * 2)
+        
+        logger.debug(f"VAD check - RMS: {rms_energy:.4f}, Peak: {peak_amplitude:.4f}, "
+                    f"Threshold: {self.silence_threshold}, Speech: {is_speech}")
+        
+        return is_speech
+    
+    def get_chunk_for_processing(self):
+        """Get audio chunk for processing"""
+        with self.lock:
+            if len(self.buffer) < self.chunk_samples:
+                return None
+            
+            logger.debug(f"Returning processing signal, buffer size: {len(self.buffer)}")
+            return np.array([1])  # Signal that chunk is ready
+    
+    def get_all_audio(self):
+        """Get all accumulated audio"""
+        with self.lock:
+            audio_copy = self.total_audio.copy()
+            logger.debug(f"Returning {len(audio_copy)} total audio samples")
+            return audio_copy
+    
+    def clear(self):
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+            logger.debug("Audio buffer cleared")
+    
+    def reset_for_new_segment(self):
+        """Reset buffers for new transcription segment"""
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+            logger.debug("Audio buffer reset for new segment")
+
+def linear16_to_audio(audio_bytes, sample_rate=8000):
+    """Convert LINEAR16 PCM bytes to numpy array"""
+    try:
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16)
+        audio_array = audio_array.astype(np.float32) / 32768.0
+        return audio_array
+    except Exception as e:
+        logger.error(f"Error converting LINEAR16 to audio: {e}")
+        return np.array([], dtype=np.float32)
+
+def resample_audio(audio_data, source_rate, target_rate):
+    """Resample audio to target sample rate"""
+    if source_rate == target_rate:
+        return audio_data
+    
+    if source_rate == 8000 and target_rate == 16000:
+        # Simple 2x upsampling for common case
+        upsampled = np.repeat(audio_data, 2)
+        return upsampled.astype(np.float32)
+    
+    # Fallback: Linear interpolation resampling
+    ratio = target_rate / source_rate
+    indices = np.arange(0, len(audio_data), 1/ratio)
+    indices = indices[indices < len(audio_data)]
+    resampled = np.interp(indices, np.arange(len(audio_data)), audio_data)
+    
+    return resampled.astype(np.float32)
+
+def transcribe_with_nemo(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio using NeMo FastConformer"""
+    try:
+        if len(audio_data) == 0 or asr_model_nemo is None:
+            return ""
+        
+        # Resample to 16kHz (NeMo models typically expect 16kHz)
+        resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        # --- Denoiser added ---
+        resampled_audio = denoise_audio(resampled_audio, sample_rate=target_sample_rate)
+        # Skip very short audio
+        min_samples = int(0.3 * target_sample_rate)
+        if len(resampled_audio) < min_samples:
+            return ""
+        
+        start_time = time.time()
+        
+        # Save audio to temporary file (NeMo expects file path)
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
+            sf.write(tmp_file.name, resampled_audio, target_sample_rate)
+            tmp_path = tmp_file.name
+        
+        try:
+            # Transcribe with NeMo
+            result = asr_model_nemo.transcribe([tmp_path])
+            
+            if result and len(result) > 0:
+                # Handle different NeMo result formats
+                if hasattr(result[0], 'text'):
+                    raw_text = result[0].text
+                elif isinstance(result[0], str):
+                    raw_text = result[0]
+                else:
+                    raw_text = str(result[0])
+                
+                if not isinstance(raw_text, str):
+                    raw_text = str(raw_text)
+                
+                if raw_text and raw_text.strip():
+                    # Convert Arabic numbers to digits for NeMo
+                    cleaned_text = replace_arabic_numbers_nemo(raw_text)
+                    end_time = time.time()
+                    
+                    if cleaned_text.strip():
+                        logger.info(f"NeMo transcription: '{cleaned_text}' (processed in {end_time - start_time:.2f}s)")
+                    
+                    return cleaned_text.strip()
+                    
+        finally:
+            # Clean up temporary file
+            if os.path.exists(tmp_path):
+                os.remove(tmp_path)
+        
+        return ""
+        
+    except Exception as e:
+        logger.error(f"Error during NeMo transcription: {e}")
+        return ""
+
+def transcribe_with_whisper(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio chunk using Whisper model directly"""
+    try:
+        if len(audio_data) == 0 or whisper_model is None:
+            return ""
+        
+        # Resample from 8kHz to 16kHz for Whisper
+        resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        
+        # Ensure minimum length for Whisper
+        min_samples = int(0.1 * target_sample_rate)  # 100ms minimum
+        if len(resampled_audio) < min_samples:
+            return ""
+        
+        start_time = time.time()
+        
+        # Prepare input features with proper dtype
+        input_features = whisper_processor(
+            resampled_audio, 
+            sampling_rate=target_sample_rate, 
+            return_tensors="pt"
+        ).input_features
+        
+        # Ensure correct dtype and device
+        input_features = input_features.to(device=device, dtype=torch_dtype)
+        
+        # Create attention mask to avoid warnings
+        attention_mask = torch.ones(
+            input_features.shape[:-1], 
+            dtype=torch.long, 
+            device=device
+        )
+        
+        # Generate transcription using model directly
+        with torch.no_grad():
+            predicted_ids = whisper_model.generate(
+                input_features,
+                attention_mask=attention_mask,
+                max_new_tokens=128,
+                do_sample=False,
+                # temperature=0.0,
+                num_beams=1,
+                language="english",
+                task="translate",
+                pad_token_id=whisper_tokenizer.pad_token_id,
+                eos_token_id=whisper_tokenizer.eos_token_id
+            )
+        
+        # Decode the transcription
+        transcription = whisper_tokenizer.batch_decode(
+            predicted_ids, 
+            skip_special_tokens=True
+        )[0].strip()
+        
+        end_time = time.time()
+        
+        logger.info(f"Whisper transcription completed in {end_time - start_time:.2f}s: '{transcription}'")
+        return transcription
+        
+    except Exception as e:
+        logger.error(f"Error during Whisper transcription: {e}")
+        return ""
+
+class UnifiedSTTHandler:
+    def __init__(self, websocket):
+        self.websocket = websocket
+        self.audio_buffer = None
+        self.config = {}
+        self.running = False
+        self.transcription_task = None
+        self.use_nemo = False  # Flag to determine which model to use
+        
+        # Auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.silence_timeout = 2.9
+        self.min_interim_count = 1
+        self.auto_final_task = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.segment_number = 0
+        self.last_partial = ""
+        
+        # Processing tracking
+        self.processing_count = 0
+
+    # Add this debugging method to your UnifiedSTTHandler class
+
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer with enhanced debugging"""
+        if self.audio_buffer and self.running:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+            
+            model_name = "NeMo" if self.use_nemo else "Whisper"
+            
+            # Debug logging every few audio packets
+            if len(audio_data) > 0:
+                total_samples = len(self.audio_buffer.get_all_audio())
+                total_seconds = total_samples / self.config["sample_rate"]
+                
+                # Log every second of audio
+                if int(total_seconds) != getattr(self, '_last_logged_second', -1):
+                    logger.info(f"{model_name} - Accumulated {total_seconds:.1f}s of audio ({total_samples} samples)")
+                    self._last_logged_second = int(total_seconds)
+                    
+                    # Check if we should have chunks ready
+                    chunk_ready = self.audio_buffer.has_chunk_ready()
+                    logger.info(f"{model_name} - Chunk ready: {chunk_ready}")        
+    # async def start_processing(self, start_message):
+    #     """Initialize with start message from jambonz"""
+    #     self.config = {
+    #         "language": start_message.get("language", "ar-EG"),
+    #         "format": start_message.get("format", "raw"),
+    #         "encoding": start_message.get("encoding", "LINEAR16"),
+    #         "sample_rate": start_message.get("sampleRateHz", 8000),
+    #         "interim_results": True,  # Always enable for internal processing
+    #         "options": start_message.get("options", {})
+    #     }
+        
+    #     # Determine which model to use based on language parameter
+    #     language = self.config["language"]
+    #     if language == "ar-EG":
+    #         logger.info("nemooooooooooooooooooooooooooo")
+    #         self.use_nemo = True
+    #         model_name = "NeMo FastConformer"
+    #     elif language == "ar-EG-whis":
+    #         logger.info("whisperrrrrrrrrrrrrrrrrrrrrrrrrrrrr")
+    #         self.use_nemo = False
+    #         model_name = "Whisper large-v3"
+    #     else:
+    #         # Default to NeMo for any other Arabic variant
+    #         self.use_nemo = True
+    #         model_name = "NeMo FastConformer (default)"
+        
+    #     logger.info(f"STT session started with {model_name} for language: {language}")
+    #     logger.info(f"Config: {self.config}")
+        
+    #     # Check if selected model is available
+    #     if self.use_nemo and asr_model_nemo is None:
+    #         await self.send_error("NeMo model not available")
+    #         return
+    #     elif not self.use_nemo and whisper_model is None:
+    #         await self.send_error("Whisper model not available")
+    #         return
+        
+    #     # Initialize audio buffer
+    #     self.audio_buffer = JambonzAudioBuffer(
+    #         sample_rate=self.config["sample_rate"],
+    #         chunk_duration=1.0  # 1 second chunks
+    #     )
+        
+    #     # Reset session variables
+    #     self.running = True
+    #     self.interim_count = 0
+    #     self.last_interim_time = None
+    #     self.accumulated_transcript = ""
+    #     self.final_sent = False
+    #     self.segment_number = 0
+    #     self.processing_count = 0
+    #     self.last_partial = ""
+        
+    #     # Start background transcription task
+    #     self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+    #     # Start auto-final detection task
+    #     self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+    
+    # Replace these methods in your UnifiedSTTHandler class
+
+    async def start_processing(self, start_message):
+        """Initialize with start message from jambonz"""
+        self.config = {
+            "language": start_message.get("language", "ar-EG"),
+            "format": start_message.get("format", "raw"),
+            "encoding": start_message.get("encoding", "LINEAR16"),
+            "sample_rate": start_message.get("sampleRateHz", 8000),
+            "interim_results": True,  # Always enable for internal processing
+            "options": start_message.get("options", {})
+        }
+        
+        # Determine which model to use based on language parameter
+        language = self.config["language"]
+        if language == "ar-EG":
+            logger.info("Selected NeMo FastConformer")
+            self.use_nemo = True
+            model_name = "NeMo FastConformer"
+        elif language == "ar-EG-whis":
+            logger.info("Selected Whisper large-v3")
+            self.use_nemo = False
+            model_name = "Whisper large-v3"
+        else:
+            # Default to NeMo for any other Arabic variant
+            self.use_nemo = True
+            model_name = "NeMo FastConformer (default)"
+        
+        logger.info(f"STT session started with {model_name} for language: {language}")
+        logger.info(f"Config: {self.config}")
+        
+        # Check if selected model is available
+        if self.use_nemo and asr_model_nemo is None:
+            await self.send_error("NeMo model not available")
+            return
+        elif not self.use_nemo and whisper_model is None:
+            await self.send_error("Whisper model not available")
+            return
+        
+        # Initialize audio buffer with model-specific settings
+        if self.use_nemo:
+            chunk_duration = 1.0  # NeMo processes every 1 second
+        else:
+            chunk_duration = 2.0  # Whisper processes every 2 seconds for better accuracy
+        
+        self.audio_buffer = JambonzAudioBuffer(
+            sample_rate=self.config["sample_rate"],
+            chunk_duration=chunk_duration
+        )
+        
+        # Adjust VAD threshold for Whisper
+        if not self.use_nemo:
+            self.audio_buffer.silence_threshold = 0.005  # Lower threshold for Whisper
+        
+        # Reset session variables
+        self.running = True
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.segment_number = 0
+        self.processing_count = 0
+        self.last_partial = ""
+        
+        # Start background transcription task
+        self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+        # Start auto-final detection task
+        self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+        
+        logger.info(f"Background tasks started for {model_name}")
+
+
+
+    async def stop_processing(self):
+        """Stop current processing session"""
+        logger.info("Stopping STT session...")
+        self.running = False
+        
+        # Cancel background tasks
+        for task in [self.transcription_task, self.auto_final_task]:
+            if task:
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+        
+        # Send final transcription if not already sent
+        if not self.final_sent and self.accumulated_transcript.strip():
+            await self.send_transcription(self.accumulated_transcript, is_final=True)
+        
+        # Process any remaining audio for comprehensive final transcription
+        if self.audio_buffer:
+            all_audio = self.audio_buffer.get_all_audio()
+            if len(all_audio) > 0 and not self.final_sent:
+                loop = asyncio.get_event_loop()
+                
+                if self.use_nemo:
+                    final_transcription = await loop.run_in_executor(
+                        executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+                    )
+                else:
+                    final_transcription = await loop.run_in_executor(
+                        executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+                    )
+                
+                if final_transcription.strip():
+                    await self.send_transcription(final_transcription, is_final=True)
+        
+        # Clear audio buffer
+        if self.audio_buffer:
+            self.audio_buffer.clear()
+        
+        logger.info("STT session stopped")
+    
+    async def start_new_segment(self):
+        """Start a new transcription segment"""
+        self.segment_number += 1
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.last_partial = ""
+        self.processing_count = 0
+        
+        if self.audio_buffer:
+            self.audio_buffer.reset_for_new_segment()
+        
+        logger.info(f"Started new transcription segment #{self.segment_number}")
+    
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer"""
+        if self.audio_buffer and self.running:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+    
+    # async def _process_audio_chunks(self):
+    #     """Process audio chunks for interim results"""
+    #     while self.running:
+    #         try:
+    #             if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+    #                 chunk_signal = self.audio_buffer.get_chunk_for_processing()
+    #                 if chunk_signal is not None:
+    #                     all_audio = self.audio_buffer.get_all_audio()
+                        
+    #                     if len(all_audio) > 0 and self.audio_buffer.is_speech(all_audio[-self.audio_buffer.chunk_samples:]):
+    #                         loop = asyncio.get_event_loop()
+                            
+    #                         # Choose transcription method based on model selection
+    #                         if self.use_nemo:
+    #                             transcription = await loop.run_in_executor(
+    #                                 executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+    #                             )
+    #                         else:
+    #                             transcription = await loop.run_in_executor(
+    #                                 executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+    #                             )
+                            
+    #                         if transcription.strip():
+    #                             self.processing_count += 1
+    #                             self.accumulated_transcript = transcription
+                                
+    #                             if transcription != self.last_partial or self.interim_count == 0:
+    #                                 self.last_partial = transcription
+    #                                 self.interim_count += 1
+    #                                 self.last_interim_time = time.time()
+    #                                 logger.info(f"Updated interim_count to {self.interim_count} for transcript: '{transcription}'")
+    #                             else:
+    #                                 self.last_interim_time = time.time()
+                
+    #             await asyncio.sleep(0.1)  # Check every 100ms
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in chunk processing: {e}")
+    #             await asyncio.sleep(0.1)
+    
+    
+    # async def _monitor_for_auto_final(self):
+    #     """Monitor for auto-final conditions"""
+    #     while self.running:
+    #         try:
+    #             current_time = time.time()
+                
+    #             if (self.interim_count >= self.min_interim_count and 
+    #                 self.last_interim_time is not None and 
+    #                 (current_time - self.last_interim_time) >= self.silence_timeout and
+    #                 not self.final_sent and
+    #                 self.accumulated_transcript.strip()):
+                    
+    #                 logger.info(f"Auto-final triggered for segment #{self.segment_number}")
+                    
+    #                 await self.send_transcription(self.accumulated_transcript, is_final=True)
+    #                 await self.start_new_segment()
+                
+    #             await asyncio.sleep(0.5)  # Check every 500ms
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in auto-final monitoring: {e}")
+    #             await asyncio.sleep(0.5)
+    
+    # async def _process_audio_chunks(self):
+    #     """Process audio chunks for interim results - FIXED for Whisper streaming"""
+    #     logger.info(f"Starting audio chunk processing for {'NeMo' if self.use_nemo else 'Whisper'}")
+        
+    #     while self.running:
+    #         try:
+    #             if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+    #                 chunk_signal = self.audio_buffer.get_chunk_for_processing()
+    #                 if chunk_signal is not None:
+    #                     all_audio = self.audio_buffer.get_all_audio()
+                        
+    #                     # Check if we have enough audio and speech activity
+    #                     if len(all_audio) > 0:
+    #                         # Get the latest chunk for VAD check
+    #                         latest_chunk_start = max(0, len(all_audio) - self.audio_buffer.chunk_samples)
+    #                         latest_chunk = all_audio[latest_chunk_start:]
+                            
+    #                         # For debugging
+    #                         logger.debug(f"Audio buffer size: {len(all_audio)} samples, Latest chunk: {len(latest_chunk)} samples")
+                            
+    #                         if self.audio_buffer.is_speech(latest_chunk):
+    #                             logger.info(f"Speech detected, processing with {'NeMo' if self.use_nemo else 'Whisper'}")
+                                
+    #                             loop = asyncio.get_event_loop()
+                                
+    #                             # Choose transcription method based on model selection
+    #                             if self.use_nemo:
+    #                                 transcription = await loop.run_in_executor(
+    #                                     executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+    #                                 )
+    #                             else:
+    #                                 # For Whisper, ensure we process the accumulated audio
+    #                                 transcription = await loop.run_in_executor(
+    #                                     executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+    #                                 )
+                                
+    #                             logger.info(f"Transcription result: '{transcription}'")
+                                
+    #                             if transcription.strip():
+    #                                 self.processing_count += 1
+    #                                 self.accumulated_transcript = transcription
+                                    
+    #                                 if transcription != self.last_partial or self.interim_count == 0:
+    #                                     self.last_partial = transcription
+    #                                     self.interim_count += 1
+    #                                     self.last_interim_time = time.time()
+    #                                     logger.info(f"Updated interim_count to {self.interim_count} for transcript: '{transcription}'")
+    #                                 else:
+    #                                     self.last_interim_time = time.time()
+    #                                     logger.info("Same transcription, updating time only")
+    #                         else:
+    #                             logger.debug("No speech detected in latest chunk")
+                
+    #             await asyncio.sleep(0.1)  # Check every 100ms
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in chunk processing: {e}")
+    #             import traceback
+    #             traceback.print_exc()
+    #             await asyncio.sleep(0.1)
+    
+    # async def _monitor_for_auto_final(self):
+    #     """Monitor for auto-final conditions - Enhanced logging"""
+    #     logger.info("Starting auto-final monitoring")
+        
+    #     while self.running:
+    #         try:
+    #             current_time = time.time()
+                
+    #             if (self.interim_count >= self.min_interim_count and 
+    #                 self.last_interim_time is not None and 
+    #                 (current_time - self.last_interim_time) >= self.silence_timeout and
+    #                 not self.final_sent and
+    #                 self.accumulated_transcript.strip()):
+                    
+    #                 silence_duration = current_time - self.last_interim_time
+    #                 logger.info(f"Auto-final triggered for segment #{self.segment_number} - "
+    #                           f"Interim count: {self.interim_count}, Silence: {silence_duration:.1f}s")
+                    
+    #                 await self.send_transcription(self.accumulated_transcript, is_final=True)
+    #                 await self.start_new_segment()
+                
+    #             # Debug logging every 5 seconds
+    #             if int(current_time) % 5 == 0:
+    #                 logger.debug(f"Auto-final status - Interim count: {self.interim_count}, "
+    #                            f"Last interim: {self.last_interim_time}, "
+    #                            f"Final sent: {self.final_sent}, "
+    #                            f"Transcript: '{self.accumulated_transcript[:50]}...'")
+                
+    #             await asyncio.sleep(0.5)  # Check every 500ms
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in auto-final monitoring: {e}")
+    #             await asyncio.sleep(0.5)
+    
+    # async def _process_audio_chunks(self):
+    #     """Process audio chunks for interim results - FIXED for both models"""
+    #     model_name = "NeMo" if self.use_nemo else "Whisper"
+    #     logger.info(f"Starting audio chunk processing for {model_name}")
+        
+    #     while self.running:
+    #         try:
+    #             if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+    #                 chunk_signal = self.audio_buffer.get_chunk_for_processing()
+    #                 if chunk_signal is not None:
+    #                     all_audio = self.audio_buffer.get_all_audio()
+                        
+    #                     # Debug logging
+    #                     logger.debug(f"Processing chunk - Total audio: {len(all_audio)} samples")
+                        
+    #                     if len(all_audio) > 0:
+    #                         # Get the latest chunk for VAD check
+    #                         latest_chunk_start = max(0, len(all_audio) - self.audio_buffer.chunk_samples)
+    #                         latest_chunk = all_audio[latest_chunk_start:]
+                            
+    #                         # Check for speech activity
+    #                         has_speech = self.audio_buffer.is_speech(latest_chunk)
+    #                         logger.debug(f"Speech detection result: {has_speech}")
+                            
+    #                         if has_speech:
+    #                             logger.info(f"Processing audio with {model_name} - {len(all_audio)} samples")
+                                
+    #                             loop = asyncio.get_event_loop()
+    #                             start_time = time.time()
+                                
+    #                             try:
+    #                                 # Choose transcription method based on model selection
+    #                                 if self.use_nemo:
+    #                                     transcription = await loop.run_in_executor(
+    #                                         executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+    #                                     )
+    #                                 else:
+    #                                     # For Whisper, ensure we have enough audio
+    #                                     if len(all_audio) >= int(0.5 * 16000):  # At least 0.5 seconds at 16kHz
+    #                                         transcription = await loop.run_in_executor(
+    #                                             executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+    #                                         )
+    #                                     else:
+    #                                         transcription = ""
+    #                                         logger.debug("Whisper: Not enough audio for transcription")
+                                    
+    #                                 process_time = time.time() - start_time
+    #                                 logger.info(f"{model_name} processing took {process_time:.2f}s, result: '{transcription}'")
+                                    
+    #                                 if transcription and transcription.strip():
+    #                                     self.processing_count += 1
+    #                                     self.accumulated_transcript = transcription
+                                        
+    #                                     if transcription != self.last_partial or self.interim_count == 0:
+    #                                         self.last_partial = transcription
+    #                                         self.interim_count += 1
+    #                                         self.last_interim_time = time.time()
+    #                                         logger.info(f"Updated interim_count to {self.interim_count} for transcript: '{transcription}'")
+    #                                     else:
+    #                                         self.last_interim_time = time.time()
+    #                                         logger.debug("Same transcription, updating time only")
+    #                                 else:
+    #                                     logger.debug(f"{model_name} returned empty transcription")
+                                        
+    #                             except Exception as e:
+    #                                 logger.error(f"Error in {model_name} transcription: {e}")
+    #                         else:
+    #                             logger.debug("No speech detected in latest chunk")
+                
+    #             # Different sleep intervals for different models
+    #             sleep_interval = 0.1 if self.use_nemo else 0.2  # Whisper can be less frequent
+    #             await asyncio.sleep(sleep_interval)
+                
+    #         except Exception as e:
+    #             logger.error(f"Error in chunk processing: {e}")
+    #             import traceback
+    #             traceback.print_exc()
+    #             await asyncio.sleep(1)  # Longer sleep on error
+
+    # Also add this to the beginning of _process_audio_chunks method:
+    
+    async def _process_audio_chunks(self):
+        """Process audio chunks for interim results - with debugging"""
+        model_name = "NeMo" if self.use_nemo else "Whisper"
+        logger.info(f"Starting audio chunk processing for {model_name}")
+        
+        chunk_count = 0
+        
+        while self.running:
+            try:
+                if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+                    chunk_count += 1
+                    logger.info(f"{model_name} - Processing chunk #{chunk_count}")
+                    
+                    chunk_signal = self.audio_buffer.get_chunk_for_processing()
+                    if chunk_signal is not None:
+                        all_audio = self.audio_buffer.get_all_audio()
+                        
+                        logger.info(f"{model_name} - Got {len(all_audio)} samples for processing")
+                        
+                        if len(all_audio) > 0:
+                            # Get the latest chunk for VAD check
+                            latest_chunk_start = max(0, len(all_audio) - self.audio_buffer.chunk_samples)
+                            latest_chunk = all_audio[latest_chunk_start:]
+                            
+                            # Check for speech activity
+                            has_speech = self.audio_buffer.is_speech(latest_chunk)
+                            logger.info(f"{model_name} - Speech detected: {has_speech}")
+                            
+                            if has_speech:
+                                logger.info(f"{model_name} - Starting transcription...")
+                                
+                                loop = asyncio.get_event_loop()
+                                start_time = time.time()
+                                
+                                try:
+                                    # Choose transcription method based on model selection
+                                    if self.use_nemo:
+                                        transcription = await loop.run_in_executor(
+                                            executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+                                        )
+                                    else:
+                                        transcription = await loop.run_in_executor(
+                                            executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+                                        )
+                                    
+                                    process_time = time.time() - start_time
+                                    logger.info(f"{model_name} - Transcription completed in {process_time:.2f}s: '{transcription}'")
+                                    
+                                    if transcription and transcription.strip():
+                                        self.processing_count += 1
+                                        self.accumulated_transcript = transcription
+                                        
+                                        if transcription != self.last_partial or self.interim_count == 0:
+                                            self.last_partial = transcription
+                                            self.interim_count += 1
+                                            self.last_interim_time = time.time()
+                                            logger.info(f"{model_name} - Updated interim_count to {self.interim_count}")
+                                        else:
+                                            self.last_interim_time = time.time()
+                                            logger.info(f"{model_name} - Same transcription, updating time only")
+                                    else:
+                                        logger.info(f"{model_name} - No transcription result")
+                                        
+                                except Exception as e:
+                                    logger.error(f"{model_name} - Transcription error: {e}")
+                                    import traceback
+                                    traceback.print_exc()
+                            else:
+                                logger.debug(f"{model_name} - No speech in chunk")
+                    else:
+                        logger.warning(f"{model_name} - Chunk signal was None")
+                else:
+                    # Log why chunk is not ready
+                    if self.audio_buffer:
+                        current_size = len(self.audio_buffer.buffer)
+                        required_size = self.audio_buffer.chunk_samples
+                        if current_size > 0:
+                            logger.debug(f"{model_name} - Buffer: {current_size}/{required_size} samples")
+                
+                await asyncio.sleep(0.1)
+                
+            except Exception as e:
+                logger.error(f"{model_name} - Error in chunk processing: {e}")
+                import traceback
+                traceback.print_exc()
+                await asyncio.sleep(1)
+
+    async def _monitor_for_auto_final(self):
+        """Monitor for auto-final conditions with model-specific timeouts"""
+        model_name = "NeMo" if self.use_nemo else "Whisper"
+        timeout = 2.0 if self.use_nemo else 3.0  # Longer timeout for Whisper
+        
+        logger.info(f"Starting auto-final monitoring for {model_name} (timeout: {timeout}s)")
+        
+        while self.running:
+            try:
+                current_time = time.time()
+                
+                if (self.interim_count >= self.min_interim_count and 
+                    self.last_interim_time is not None and 
+                    (current_time - self.last_interim_time) >= timeout and
+                    not self.final_sent and
+                    self.accumulated_transcript.strip()):
+                    
+                    silence_duration = current_time - self.last_interim_time
+                    logger.info(f"Auto-final triggered for segment #{self.segment_number} ({model_name}) - "
+                            f"Interim count: {self.interim_count}, Silence: {silence_duration:.1f}s")
+                    
+                    await self.send_transcription(self.accumulated_transcript, is_final=True)
+                    await self.start_new_segment()
+                
+                await asyncio.sleep(0.5)  # Check every 500ms
+                
+            except Exception as e:
+                logger.error(f"Error in auto-final monitoring: {e}")
+                await asyncio.sleep(0.5)
+
+
+
+    async def send_transcription(self, text, is_final=True, confidence=0.9):
+        """Send transcription in jambonz format"""
+        try:
+            # Apply number conversion only for Whisper
+            if not self.use_nemo and is_final:
+                original_text = text
+                converted_text = convert_arabic_numbers_whisper(text)
+                
+                if original_text != converted_text:
+                    logger.info(f"Whisper - Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+                text = converted_text
+            
+            message = {
+                "type": "transcription",
+                "is_final": True,  # Always send as final
+                "alternatives": [
+                    {
+                        "transcript": text,
+                        "confidence": confidence
+                    }
+                ],
+                "language": self.config.get("language", "ar-EG"),
+                "channel": 1
+            }
+            
+            await self.websocket.send(json.dumps(message))
+            self.final_sent = True
+            
+            model_name = "NeMo" if self.use_nemo else "Whisper"
+            logger.info(f"Sent FINAL transcription ({model_name}): '{text}'")
+            
+        except Exception as e:
+            logger.error(f"Error sending transcription: {e}")
+    
+    async def send_error(self, error_message):
+        """Send error message in jambonz format"""
+        try:
+            message = {
+                "type": "error",
+                "error": error_message
+            }
+            await self.websocket.send(json.dumps(message))
+            logger.error(f"Sent error: {error_message}")
+        except Exception as e:
+            logger.error(f"Error sending error message: {e}")
+
+async def handle_jambonz_websocket(websocket):
+    """Handle jambonz WebSocket connections"""
+    
+    client_id = f"jambonz_{id(websocket)}"
+    logger.info(f"New unified STT connection: {client_id}")
+    
+    handler = UnifiedSTTHandler(websocket)
+    
+    try:
+        async for message in websocket:
+            try:
+                if isinstance(message, str):
+                    data = json.loads(message)
+                    message_type = data.get("type")
+                    
+                    if message_type == "start":
+                        logger.info(f"Received start message: {data}")
+                        await handler.start_processing(data)
+                        
+                    elif message_type == "stop":
+                        logger.info("Received stop message - closing WebSocket")
+                        await handler.stop_processing()
+                        await websocket.close(code=1000, reason="Session stopped by client")
+                        break
+                        
+                    else:
+                        logger.warning(f"Unknown message type: {message_type}")
+                        await handler.send_error(f"Unknown message type: {message_type}")
+                
+                else:
+                    # Handle binary audio data
+                    if not handler.running or handler.audio_buffer is None:
+                        logger.warning("Received audio data outside of active session")
+                        await handler.send_error("Received audio before start message or after stop")
+                        continue
+                    
+                    await handler.add_audio_data(message)
+            
+            except json.JSONDecodeError as e:
+                logger.error(f"JSON decode error: {e}")
+                await handler.send_error(f"Invalid JSON: {str(e)}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+                await handler.send_error(f"Processing error: {str(e)}")
+    
+    except websockets.exceptions.ConnectionClosed:
+        logger.info(f"Unified STT connection closed: {client_id}")
+    except Exception as e:
+        logger.error(f"Unified STT WebSocket error: {e}")
+        try:
+            await handler.send_error(str(e))
+        except:
+            pass
+    finally:
+        if handler.running:
+            await handler.stop_processing()
+        logger.info(f"Unified STT connection ended: {client_id}")
+
+async def main():
+    """Start the Unified Arabic STT WebSocket server"""
+    logger.info("Starting Unified Arabic STT WebSocket server on port 3007...")
+    
+    # Check model availability
+    models_available = []
+    if asr_model_nemo is not None:
+        models_available.append("NeMo FastConformer (ar-EG)")
+    if whisper_model is not None:
+        models_available.append("Whisper large-v3 (ar-EG-whis)")
+    
+    if not models_available:
+        logger.error("No models available! Please check model paths and installations.")
+        return
+    
+    # Start WebSocket server
+    server = await websockets.serve(
+        handle_jambonz_websocket,
+        "0.0.0.0", 
+        3007,
+        ping_interval=20,
+        ping_timeout=10,
+        close_timeout=10
+    )
+    
+    logger.info("Unified Arabic STT WebSocket server started on ws://0.0.0.0:3007")
+    logger.info("Ready to handle jambonz STT requests with both models")
+    logger.info("ROUTING:")
+    logger.info("- language: 'ar-EG' → NeMo FastConformer (with built-in number conversion)")
+    logger.info("- language: 'ar-EG-whis' → Whisper large-v3 (with pyarabic number conversion)")
+    logger.info("FEATURES:")
+    logger.info("- Continuous transcription with segmentation")
+    logger.info("- Voice Activity Detection")
+    logger.info("- Auto-final detection (2s silence timeout)")
+    logger.info("- Model-specific number conversion")
+    logger.info(f"AVAILABLE MODELS: {', '.join(models_available)}")
+    
+    # Wait for the server to close
+    await server.wait_closed()
+
+if __name__ == "__main__":
+    print("=" * 80)
+    print("Unified Arabic STT Server (NeMo + Whisper)")
+    print("=" * 80)
+    print("WebSocket Port: 3007")
+    print("Protocol: jambonz STT API")
+    print("Audio Format: LINEAR16 PCM @ 8kHz → 16kHz")
+    print()
+    print("LANGUAGE ROUTING:")
+    print("- 'ar-EG' → NeMo FastConformer")
+    print("  • Built-in Arabic number word to digit conversion")
+    print("  • Optimized for Arabic dialects")
+    print("- 'ar-EG-whis' → Whisper large-v3")
+    print("  • pyarabic library number conversion (final transcripts only)")
+    print("  • OpenAI Whisper model")
+    print()
+    print("FEATURES:")
+    print("- Automatic model selection based on language parameter")
+    print("- Voice Activity Detection")
+    print("- Auto-final detection (2 seconds silence)")
+    print("- Model-specific number conversion strategies")
+    print("- Continuous transcription with segmentation")
+    print()
+    
+    # Check model availability for startup info
+    nemo_status = "✓ Available" if asr_model_nemo is not None else "✗ Not Available"
+    whisper_status = "✓ Available" if whisper_model is not None else "✗ Not Available"
+    arabic_numbers_status = "✓ Available" if arabic_numbers_available else "✗ Not Available (install pyarabic)"
+    
+    print("MODEL STATUS:")
+    print(f"- NeMo FastConformer: {nemo_status}")
+    print(f"- Whisper large-v3: {whisper_status}")
+    print(f"- pyarabic (Whisper numbers): {arabic_numbers_status}")
+    print("=" * 80)
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nShutting down unified server...")
+    except Exception as e:
+        print(f"Server error: {e}")

denoiser_model.py

@@ -0,0 +1,8 @@
+import torch
+from denoiser import pretrained
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# Load DNS64 pretrained model (auto-downloads if not cached)
+denoiser_model = pretrained.dns64().to(device)
+denoiser_model.eval()

improved_asr_web_ui.html

@@ -0,0 +1,729 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>ASR WebSocket Testing Client with Sample Rate Analysis</title>
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+
+        body {
+            font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            min-height: 100vh;
+            padding: 20px;
+        }
+
+        .container {
+            background: rgba(255, 255, 255, 0.95);
+            backdrop-filter: blur(10px);
+            border-radius: 20px;
+            padding: 40px;
+            box-shadow: 0 20px 40px rgba(0, 0, 0, 0.1);
+            max-width: 800px;
+            margin: 0 auto;
+            border: 1px solid rgba(255, 255, 255, 0.2);
+        }
+
+        .header {
+            text-align: center;
+            margin-bottom: 30px;
+        }
+
+        .header h1 {
+            color: #333;
+            font-size: 2.5em;
+            font-weight: 300;
+            margin-bottom: 10px;
+        }
+
+        .header p {
+            color: #666;
+            font-size: 1.1em;
+        }
+
+        .connection-section {
+            margin-bottom: 30px;
+        }
+
+        .input-group {
+            margin-bottom: 20px;
+        }
+
+        .input-group label {
+            display: block;
+            margin-bottom: 8px;
+            color: #333;
+            font-weight: 500;
+        }
+
+        .input-group input, .input-group select {
+            width: 100%;
+            padding: 12px 16px;
+            border: 2px solid #e1e5e9;
+            border-radius: 10px;
+            font-size: 16px;
+            transition: all 0.3s ease;
+            background: rgba(255, 255, 255, 0.8);
+        }
+
+        .input-group input:focus, .input-group select:focus {
+            outline: none;
+            border-color: #667eea;
+            box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1);
+        }
+
+        .btn {
+            padding: 12px 24px;
+            border: none;
+            border-radius: 10px;
+            font-size: 16px;
+            font-weight: 500;
+            cursor: pointer;
+            transition: all 0.3s ease;
+            text-transform: uppercase;
+            letter-spacing: 0.5px;
+        }
+
+        .btn:disabled {
+            opacity: 0.6;
+            cursor: not-allowed;
+        }
+
+        .btn-connect {
+            background: linear-gradient(135deg, #4CAF50, #45a049);
+            color: white;
+            width: 100%;
+        }
+
+        .btn-connect:hover:not(:disabled) {
+            transform: translateY(-2px);
+            box-shadow: 0 5px 15px rgba(76, 175, 80, 0.3);
+        }
+
+        .btn-disconnect {
+            background: linear-gradient(135deg, #f44336, #da190b);
+            color: white;
+            width: 100%;
+        }
+
+        .audio-controls {
+            display: flex;
+            justify-content: center;
+            gap: 20px;
+            margin: 30px 0;
+        }
+
+        .btn-mic {
+            background: linear-gradient(135deg, #2196F3, #1976D2);
+            color: white;
+            width: 80px;
+            height: 80px;
+            border-radius: 50%;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            font-size: 24px;
+        }
+
+        .btn-mic:hover:not(:disabled) {
+            transform: scale(1.1);
+            box-shadow: 0 10px 25px rgba(33, 150, 243, 0.3);
+        }
+
+        .btn-mic.recording {
+            background: linear-gradient(135deg, #f44336, #da190b);
+            animation: pulse 1.5s infinite;
+        }
+
+        .btn-stop {
+            background: linear-gradient(135deg, #FF9800, #F57C00);
+            color: white;
+            width: 80px;
+            height: 80px;
+            border-radius: 50%;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            font-size: 24px;
+        }
+
+        @keyframes pulse {
+            0% { transform: scale(1); }
+            50% { transform: scale(1.05); }
+            100% { transform: scale(1); }
+        }
+
+        .status {
+            text-align: center;
+            margin: 20px 0;
+            padding: 12px;
+            border-radius: 10px;
+            font-weight: 500;
+        }
+
+        .status.connected {
+            background: rgba(76, 175, 80, 0.1);
+            color: #4CAF50;
+            border: 1px solid rgba(76, 175, 80, 0.3);
+        }
+
+        .status.disconnected {
+            background: rgba(244, 67, 54, 0.1);
+            color: #f44336;
+            border: 1px solid rgba(244, 67, 54, 0.3);
+        }
+
+        .status.recording {
+            background: rgba(33, 150, 243, 0.1);
+            color: #2196F3;
+            border: 1px solid rgba(33, 150, 243, 0.3);
+        }
+
+        .stats-section {
+            display: grid;
+            grid-template-columns: 1fr 1fr;
+            gap: 20px;
+            margin: 20px 0;
+        }
+
+        .stat-box {
+            background: rgba(0, 0, 0, 0.05);
+            border-radius: 10px;
+            padding: 15px;
+            text-align: center;
+            border: 1px solid rgba(0, 0, 0, 0.1);
+        }
+
+        .stat-value {
+            font-size: 2em;
+            font-weight: bold;
+            color: #667eea;
+            margin-bottom: 5px;
+        }
+
+        .stat-label {
+            color: #666;
+            font-size: 0.9em;
+        }
+
+        .response-section {
+            margin-top: 30px;
+        }
+
+        .response-box {
+            background: rgba(0, 0, 0, 0.05);
+            border-radius: 10px;
+            padding: 20px;
+            min-height: 200px;
+            max-height: 400px;
+            overflow-y: auto;
+            border: 1px solid rgba(0, 0, 0, 0.1);
+            font-family: 'Courier New', monospace;
+            font-size: 12px;
+            white-space: pre-wrap;
+            word-wrap: break-word;
+        }
+
+        .debug-section {
+            margin-top: 20px;
+        }
+
+        .debug-box {
+            background: rgba(0, 0, 0, 0.8);
+            color: #00ff00;
+            border-radius: 10px;
+            padding: 15px;
+            min-height: 150px;
+            max-height: 300px;
+            overflow-y: auto;
+            font-family: 'Courier New', monospace;
+            font-size: 11px;
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="header">
+            <h1>🎤 ASR Sample Rate Analyzer</h1>
+            <p>WebSocket-based Speech Recognition with Audio Analysis</p>
+        </div>
+
+        <div class="connection-section">
+            <div class="input-group">
+                <label for="websocketUrl">WebSocket URL:</label>
+                <input type="text" id="websocketUrl" value="ws://185.208.206.135:5005" placeholder="ws://185.208.206.135:5005">
+            </div>
+            
+            <div class="input-group">
+                <label for="targetSampleRate">Target Sample Rate (Hz):</label>
+                <select id="targetSampleRate">
+                    <option value="8000">8000 Hz (Default)</option>
+                    <option value="16000">16000 Hz</option>
+                    <option value="22050">22050 Hz</option>
+                    <option value="44100">44100 Hz</option>
+                </select>
+            </div>
+            
+            <div class="input-group">
+                <label for="chunkSize">Audio Chunk Size (samples):</label>
+                <select id="chunkSize">
+                    <option value="1024">1024 samples</option>
+                    <option value="2048">2048 samples</option>
+                    <option value="4096" selected>4096 samples</option>
+                    <option value="8192">8192 samples</option>
+                </select>
+            </div>
+
+            <div class="input-group">
+                <label for="Interim-Results"> Interim-Results:</label>
+                <select id="interimResults">
+                    <option value="true">True</option>
+                    <option value="false">False</option>
+                </select>
+            </div>
+
+            <div class="input-group">
+                <label for="language">Language Code:</label>
+                <input type="text" id="language_code" value="en-US" placeholder="en-US">
+            </div>
+
+            
+            <button id="connectBtn" class="btn btn-connect">Connect to Debug Server</button>
+            <button id="disconnectBtn" class="btn btn-disconnect" style="display: none;">Disconnect</button>
+        </div>
+
+        <div id="status" class="status disconnected">Disconnected</div>
+
+        <div class="stats-section">
+            <div class="stat-box">
+                <div class="stat-value" id="actualSampleRate">0</div>
+                <div class="stat-label">Calculated Sample Rate (Hz)</div>
+            </div>
+            <div class="stat-box">
+                <div class="stat-value" id="bytesSent">0</div>
+                <div class="stat-label">Total Bytes Sent</div>
+            </div>
+            <div class="stat-box">
+                <div class="stat-value" id="chunksSent">0</div>
+                <div class="stat-label">Audio Chunks Sent</div>
+            </div>
+            <div class="stat-box">
+                <div class="stat-value" id="duration">0.0s</div>
+                <div class="stat-label">Recording Duration</div>
+            </div>
+        </div>
+
+        <div class="audio-controls">
+            <button id="micBtn" class="btn btn-mic" disabled title="Start Recording">🎤</button>
+            <button id="stopBtn" class="btn btn-stop" disabled title="Stop Recording">⏹️</button>
+        </div>
+
+        <div class="response-section">
+            <h3>Server Responses:</h3>
+            <div id="responseBox" class="response-box">Waiting for connection...</div>
+        </div>
+
+        <div class="debug-section">
+            <h3>Debug Console:</h3>
+            <div id="debugBox" class="debug-box">Ready to connect...</div>
+        </div>
+    </div>
+
+    <script>
+        class SampleRateAnalyzer {
+            constructor() {
+                this.websocket = null;
+                this.audioContext = null;
+                this.mediaRecorder = null;
+                this.audioStream = null;
+                this.processor = null;
+                this.isRecording = false;
+                this.isConnected = false;
+                
+                // Audio analysis variables
+                this.startTime = null;
+                this.totalBytesSent = 0;
+                this.chunksSent = 0;
+                this.targetSampleRate = 8000;
+                this.chunkSize = 4096;
+                
+                this.initializeElements();
+                this.attachEventListeners();
+            }
+
+            initializeElements() {
+                this.elements = {
+                    websocketUrl: document.getElementById('websocketUrl'),
+                    targetSampleRate: document.getElementById('targetSampleRate'),
+                    chunkSize: document.getElementById('chunkSize'),
+                    connectBtn: document.getElementById('connectBtn'),
+                    disconnectBtn: document.getElementById('disconnectBtn'),
+                    micBtn: document.getElementById('micBtn'),
+                    stopBtn: document.getElementById('stopBtn'),
+                    status: document.getElementById('status'),
+                    responseBox: document.getElementById('responseBox'),
+                    debugBox: document.getElementById('debugBox'),
+                    actualSampleRate: document.getElementById('actualSampleRate'),
+                    bytesSent: document.getElementById('bytesSent'),
+                    chunksSent: document.getElementById('chunksSent'),
+                    duration: document.getElementById('duration')
+                };
+            }
+
+            attachEventListeners() {
+                this.elements.connectBtn.addEventListener('click', () => this.connect());
+                this.elements.disconnectBtn.addEventListener('click', () => this.disconnect());
+                this.elements.micBtn.addEventListener('click', () => this.startRecording());
+                this.elements.stopBtn.addEventListener('click', () => this.stopRecording());
+                this.elements.targetSampleRate.addEventListener('change', (e) => {
+                    this.targetSampleRate = parseInt(e.target.value);
+                    this.debugLog(`Target sample rate changed to: ${this.targetSampleRate} Hz`);
+                });
+                this.elements.chunkSize.addEventListener('change', (e) => {
+                    this.chunkSize = parseInt(e.target.value);
+                    this.debugLog(`Chunk size changed to: ${this.chunkSize} samples`);
+                });
+            }
+
+            debugLog(message) {
+                const timestamp = new Date().toLocaleTimeString();
+                const debugBox = this.elements.debugBox;
+                debugBox.innerHTML += `[${timestamp}] ${message}\n`;
+                debugBox.scrollTop = debugBox.scrollHeight;
+            }
+
+            updateStatus(message, type) {
+                this.elements.status.textContent = message;
+                this.elements.status.className = `status ${type}`;
+            }
+
+            updateStats() {
+                if (this.startTime) {
+                    const elapsed = (Date.now() - this.startTime) / 1000;
+                    this.elements.duration.textContent = `${elapsed.toFixed(1)}s`;
+                    
+                    // Calculate actual sample rate
+                    if (elapsed > 0.5) {
+                        const totalSamples = this.totalBytesSent / 2; // 16-bit samples
+                        const calculatedRate = totalSamples / elapsed;
+                        this.elements.actualSampleRate.textContent = Math.round(calculatedRate);
+                        
+                        // Log if there's a significant difference
+                        const difference = Math.abs(calculatedRate - this.targetSampleRate);
+                        if (difference > 100) {
+                            this.debugLog(`⚠️  Sample rate mismatch! Target: ${this.targetSampleRate}Hz, Actual: ${Math.round(calculatedRate)}Hz`);
+                        }
+                    }
+                }
+                
+                this.elements.bytesSent.textContent = this.totalBytesSent.toLocaleString();
+                this.elements.chunksSent.textContent = this.chunksSent.toLocaleString();
+            }
+
+            async connect() {
+                const url = this.elements.websocketUrl.value.trim();
+                if (!url) {
+                    alert('Please enter a WebSocket URL');
+                    return;
+                }
+
+                try {
+                    this.updateStatus('Connecting...', 'disconnected');
+                    this.elements.connectBtn.disabled = true;
+                    this.debugLog(`Attempting to connect to: ${url}`);
+
+                    this.websocket = new WebSocket(url);
+                    this.websocket.binaryType = 'arraybuffer';
+
+                    // Set a connection timeout
+                    const connectionTimeout = setTimeout(() => {
+                        if (this.websocket.readyState === WebSocket.CONNECTING) {
+                            this.websocket.close();
+                            this.debugLog('❌ Connection timeout');
+                            this.updateStatus('Connection Timeout', 'disconnected');
+                            this.resetConnection();
+                        }
+                    }, 10000); // 10 second timeout
+
+                    this.websocket.onopen = () => {
+                        clearTimeout(connectionTimeout);
+                        this.isConnected = true;
+                        this.updateStatus('Connected to Debug Server', 'connected');
+                        this.elements.connectBtn.style.display = 'none';
+                        this.elements.disconnectBtn.style.display = 'block';
+                        this.elements.micBtn.disabled = false;
+                        this.elements.responseBox.textContent = 'Connected to debug server. Ready to test sample rate...';
+                        this.debugLog('✅ WebSocket connected successfully');
+                    };
+
+                    this.websocket.onmessage = (event) => {
+                        if (typeof event.data === 'string') {
+                            try {
+                                const response = JSON.parse(event.data);
+                                this.displayResponse('JSON Response', response);
+                                this.debugLog(`📨 Received JSON: ${JSON.stringify(response)}`);
+                            } catch (e) {
+                                this.displayResponse('Text Response', event.data);
+                                this.debugLog(`📨 Received Text: ${event.data}`);
+                            }
+                        } else {
+                            this.displayResponse('Binary Response', `Received binary data: ${event.data.byteLength} bytes`);
+                            this.debugLog(`📨 Received Binary: ${event.data.byteLength} bytes`);
+                        }
+                    };
+
+                    this.websocket.onerror = (error) => {
+                        clearTimeout(connectionTimeout);
+                        console.error('WebSocket error:', error);
+                        this.debugLog(`❌ WebSocket error: ${error.message || 'Unknown error'}`);
+                        this.updateStatus('Connection Error', 'disconnected');
+                        this.resetConnection();
+                    };
+
+                    this.websocket.onclose = (event) => {
+                        clearTimeout(connectionTimeout);
+                        this.isConnected = false;
+                        const reason = event.reason || 'No reason provided';
+                        this.debugLog(`🔌 WebSocket closed: Code ${event.code}, Reason: ${reason}`);
+                        this.updateStatus(`Disconnected (Code: ${event.code})`, 'disconnected');
+                        this.resetConnection();
+                        this.displayResponse('Connection Closed', `Code: ${event.code}, Reason: ${reason}`);
+                    };
+
+                } catch (error) {
+                    console.error('Connection failed:', error);
+                    this.debugLog(`❌ Connection failed: ${error.message}`);
+                    this.updateStatus('Connection Failed', 'disconnected');
+                    this.resetConnection();
+                }
+            }
+
+            disconnect() {
+                if (this.isRecording) {
+                    this.stopRecording();
+                }
+                if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
+                    this.debugLog('🔌 Manually disconnecting...');
+                    this.websocket.close(1000, 'Client disconnect');
+                }
+                this.resetConnection();
+            }
+
+            resetConnection() {
+                this.isConnected = false;
+                this.elements.connectBtn.disabled = false;
+                this.elements.connectBtn.style.display = 'block';
+                this.elements.disconnectBtn.style.display = 'none';
+                this.elements.micBtn.disabled = true;
+                this.elements.stopBtn.disabled = true;
+                this.stopRecording();
+                
+                // Reset stats
+                this.totalBytesSent = 0;
+                this.chunksSent = 0;
+                this.startTime = null;
+                this.updateStats();
+            }
+
+            // Convert Float32Array to Int16Array (LINEAR16 PCM)
+            floatTo16BitPCM(float32Array) {
+                const int16Array = new Int16Array(float32Array.length);
+                for (let i = 0; i < float32Array.length; i++) {
+                    const clipped = Math.max(-1, Math.min(1, float32Array[i]));
+                    int16Array[i] = clipped * 0x7FFF;
+                }
+                return int16Array;
+            }
+
+            // Resample audio to target sample rate
+            resampleAudio(audioBuffer, sourceSampleRate, targetSampleRate) {
+                if (sourceSampleRate === targetSampleRate) {
+                    return audioBuffer;
+                }
+                
+                const ratio = sourceSampleRate / targetSampleRate;
+                const targetLength = Math.round(audioBuffer.length / ratio);
+                const resampled = new Float32Array(targetLength);
+                
+                for (let i = 0; i < targetLength; i++) {
+                    const sourceIndex = i * ratio;
+                    const sourceIndexFloor = Math.floor(sourceIndex);
+                    const sourceIndexCeil = Math.min(sourceIndexFloor + 1, audioBuffer.length - 1);
+                    const weight = sourceIndex - sourceIndexFloor;
+                    
+                    resampled[i] = audioBuffer[sourceIndexFloor] * (1 - weight) + 
+                                   audioBuffer[sourceIndexCeil] * weight;
+                }
+                
+                return resampled;
+            }
+
+            async startRecording() {
+                if (!this.isConnected) {
+                    alert('Please connect to debug server first');
+                    return;
+                }
+
+                try {
+                    this.debugLog(`🎤 Starting recording with target sample rate: ${this.targetSampleRate} Hz`);
+                    
+                    // Initialize audio context
+                    this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
+                    this.debugLog(`🔊 Audio context created with sample rate: ${this.audioContext.sampleRate} Hz`);
+
+                    // Get microphone stream
+                    this.audioStream = await navigator.mediaDevices.getUserMedia({ 
+                        audio: {
+                            echoCancellation: false,
+                            noiseSuppression: false,
+                            autoGainControl: false,
+                            channelCount: 1
+                        } 
+                    });
+
+                    const source = this.audioContext.createMediaStreamSource(this.audioStream);
+
+                    // Create processor with specified chunk size
+                    this.processor = this.audioContext.createScriptProcessor(this.chunkSize, 1, 1);
+                    
+                    this.processor.onaudioprocess = (event) => {
+                        if (!this.isRecording || !this.websocket || this.websocket.readyState !== WebSocket.OPEN) {
+                            return;
+                        }
+
+                        const inputBuffer = event.inputBuffer;
+                        const audioData = inputBuffer.getChannelData(0);
+                        
+                        // Resample to target sample rate
+                        const resampled = this.resampleAudio(audioData, this.audioContext.sampleRate, this.targetSampleRate);
+                        
+                        // Convert to LINEAR16 PCM
+                        const pcmData = this.floatTo16BitPCM(resampled);
+                        
+                        // Send binary audio data
+                        this.websocket.send(pcmData.buffer);
+                        
+                        // Update stats
+                        this.totalBytesSent += pcmData.buffer.byteLength;
+                        this.chunksSent += 1;
+                        this.updateStats();
+                        
+                        if (this.chunksSent % 10 === 0) { // Log every 10 chunks
+                            this.debugLog(`📊 Sent ${this.chunksSent} chunks, ${this.totalBytesSent} bytes`);
+                        }
+                    };
+
+                    // Connect audio nodes
+                    source.connect(this.processor);
+                    this.processor.connect(this.audioContext.destination);
+
+                    // Send START message
+                    const startMessage = {
+                        type: "start",
+                        language: this.language_code,
+                        format: "raw",
+                        encoding: "LINEAR16",
+                        interimResults: this.interimResults,
+                        sampleRateHz: this.targetSampleRate,
+                        options: {
+                            testClient: true,
+                            chunkSize: this.chunkSize
+                        }
+                    };
+
+                    this.websocket.send(JSON.stringify(startMessage));
+                    this.displayResponse('Sent START', startMessage);
+                    this.debugLog(`📤 Sent START message: ${JSON.stringify(startMessage)}`);
+
+                    this.isRecording = true;
+                    this.startTime = Date.now();
+
+                    // Update UI
+                    this.elements.micBtn.classList.add('recording');
+                    this.elements.micBtn.disabled = true;
+                    this.elements.stopBtn.disabled = false;
+                    this.updateStatus(`Recording @ ${this.targetSampleRate}Hz`, 'recording');
+
+                } catch (error) {
+                    console.error('Failed to start recording:', error);
+                    this.debugLog(`❌ Recording failed: ${error.message}`);
+                    alert('Failed to access microphone. Please check permissions.');
+                    this.stopRecording();
+                }
+            }
+
+            stopRecording() {
+                if (this.isRecording) {
+                    this.isRecording = false;
+                    this.debugLog('🛑 Stopping recording...');
+
+                    // Send STOP message
+                    if (this.websocket && this.websocket.readyState === WebSocket.OPEN) {
+                        const stopMessage = { type: "stop" };
+                        this.websocket.send(JSON.stringify(stopMessage));
+                        this.displayResponse('Sent STOP', stopMessage);
+                        this.debugLog(`📤 Sent STOP message`);
+                    }
+                }
+
+                // Clean up audio resources
+                if (this.processor) {
+                    this.processor.disconnect();
+                    this.processor = null;
+                }
+
+                if (this.audioContext) {
+                    this.audioContext.close().then(() => {
+                        this.audioContext = null;
+                        this.debugLog('🔊 Audio context closed');
+                    });
+                }
+
+                if (this.audioStream) {
+                    this.audioStream.getTracks().forEach(track => track.stop());
+                    this.audioStream = null;
+                    this.debugLog('🎤 Microphone stream stopped');
+                }
+
+                // Update UI
+                this.elements.micBtn.classList.remove('recording');
+                this.elements.micBtn.disabled = false;
+                this.elements.stopBtn.disabled = true;
+                
+                if (this.isConnected) {
+                    this.updateStatus('Connected - Recording stopped', 'connected');
+                }
+
+                // Final stats update
+                this.updateStats();
+            }
+
+            displayResponse(messageType, response) {
+                const responseBox = this.elements.responseBox;
+                const timestamp = new Date().toLocaleTimeString();
+                
+                let content = `[${timestamp}] ${messageType}:\n`;
+                
+                if (typeof response === 'object') {
+                    content += JSON.stringify(response, null, 2);
+                } else {
+                    content += response;
+                }
+                
+                responseBox.innerHTML += content + '\n\n';
+                responseBox.scrollTop = responseBox.scrollHeight;
+            }
+        }
+
+        // Initialize when page loads
+        document.addEventListener('DOMContentLoaded', () => {
+            new SampleRateAnalyzer();
+        });
+    </script>
+</body>
+</html>

pretrained_models/asr-whisper-large-v2-commonvoice-ar/hyperparams.yaml

@@ -0,0 +1,58 @@
+# ################################
+# Model: Whisper (Encoder-Decoder) + NLL
+# Augmentation: TimeDomainSpecAugment
+# Authors: Pooneh Mousavi 2022
+# ################################
+
+
+# URL for the biggest Fairseq english whisper model.
+whisper_hub: openai/whisper-large-v2
+
+# Normalize inputs with
+# the same normalization done in the paper. Refer to Appendix C for further information.
+normalized_transcripts: True
+
+
+language: arabic 
+
+auto_mix_prec: False
+sample_rate: 16000
+
+# Decoding parameters
+min_decode_ratio: 0.0
+max_decode_ratio: 1.0
+test_beam_size: 8
+
+# Model parameters
+freeze_whisper: True
+freeze_encoder: True
+
+
+whisper: !new:speechbrain.lobes.models.huggingface_transformers.whisper.Whisper
+    source: !ref <whisper_hub>
+    freeze: !ref <freeze_whisper>
+    freeze_encoder: !ref <freeze_encoder>
+    save_path: whisper_checkpoints
+    encoder_only:  False
+
+decoder: !new:speechbrain.decoders.seq2seq.S2SWhisperGreedySearcher
+    model: !ref <whisper>
+    min_decode_ratio: !ref <min_decode_ratio>
+    max_decode_ratio: !ref <max_decode_ratio>
+
+# test_beam_searcher: !new:speechbrain.decoders.seq2seq.S2SWhisperBeamSearcher
+#     module: [!ref <whisper>]
+#     min_decode_ratio: !ref <min_decode_ratio>
+#     max_decode_ratio: !ref <max_decode_ratio>
+#     beam_size: !ref <test_beam_size>
+
+
+modules:
+    whisper: !ref <whisper>
+    decoder:  !ref <decoder>
+ 
+
+pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
+    loadables:
+        whisper: !ref <whisper>
+

pretrained_models/asr-whisper-large-v2-commonvoice-ar/whisper.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ac653766f62d8701b6fe6177a77505f98564e6c5f4c03948f2c87ad21db18c4
+size 6173767281

requirements_denoiser.txt

@@ -0,0 +1,3 @@
+pip install git+https://github.com/facebookresearch/denoiser
+
+pip install noisereduce

speech_brain_whisper_denoiser.py

@@ -0,0 +1,741 @@
+# import torch
+# import asyncio
+# import websockets
+# import json
+# import threading
+# import numpy as np
+# from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, WhisperTokenizer, pipeline , WhisperForConditionalGeneration, WhisperProcessor
+# import subprocess
+# import logging
+# import time
+# from concurrent.futures import ThreadPoolExecutor
+# import struct
+# import re
+# 3 - 10 - 2025
+import torch
+import asyncio
+import websockets
+import json
+import threading
+import numpy as np
+from transformers import pipeline
+import subprocess
+import logging
+import time
+from concurrent.futures import ThreadPoolExecutor
+import re
+import tempfile
+import os
+import soundfile as sf
+from pathlib import Path
+# --- Denoiser added ---
+try:
+    import noisereduce as nr
+    denoiser_available = True
+    print("Denoiser available (using noisereduce)")
+except ImportError:
+    denoiser_available = False
+    print("noisereduce not available - install with: pip install noisereduce")
+##############################################################################################
+# Arabic number conversion imports
+try:
+    from pyarabic.number import text2number
+    arabic_numbers_available = True
+    print("Arabic number conversion available")
+except ImportError:
+    arabic_numbers_available = False
+    print("pyarabic not available - install with: pip install pyarabic")
+    print("Arabic numbers will not be converted to digits")
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+# 3 - 10 - 2025
+# def denoise_audio(audio_data, sample_rate=16000):
+#     """Apply noise reduction to audio using noisereduce."""
+#     if not denoiser_available or len(audio_data) == 0:
+#         return audio_data
+#     try:
+#         reduced = nr.reduce_noise(y=audio_data, sr=sample_rate)
+#         return reduced.astype(np.float32)
+#     except Exception as e:
+#         logger.warning(f"Denoiser failed: {e}")
+#         return audio_data
+#############################################################################################
+def convert_arabic_numbers_in_sentence(sentence: str) -> str:
+    """
+    Replace Arabic number words in a sentence with digits,
+    preserving all other words and punctuation.
+    Handles common spelling variants and zero explicitly.
+    """
+    try:
+        print("Fxn called--------------")
+
+        # --- Normalization step ---
+        replacements = {
+            "اربعة": "أربعة",
+            "اربع": "أربع",
+            "اثنين": "اثنان",
+            "اتنين": "اثنان",   # Egyptian variant
+            "ثلاث": "ثلاثة",
+            "خمس": "خمسة",
+            "ست": "ستة",
+            "سبع": "سبعة",
+            "ثمان": "ثمانية",
+            "تسع": "تسعة",
+            "عشر": "عشرة",
+        }
+        for wrong, correct in replacements.items():
+            sentence = re.sub(rf"\b{wrong}\b", correct, sentence)
+
+        # --- Split by whitespace but keep spaces ---
+        words = re.split(r'(\s+)', sentence)
+        converted_words = []
+
+        for word in words:
+            stripped = word.strip()
+            if not stripped:  # skip spaces
+                converted_words.append(word)
+                continue
+
+            try:
+                num = text2number(stripped)
+
+                # Accept valid numbers, including zero explicitly
+                if isinstance(num, int):
+                    if num != 0 or stripped == "صفر":
+                        converted_words.append(str(num))
+                    else:
+                        converted_words.append(word)
+                else:
+                    converted_words.append(word)
+
+            except Exception:
+                converted_words.append(word)
+
+        return ''.join(converted_words)
+
+    except Exception as e:
+        logger.warning(f"Error converting Arabic numbers: {e}")
+        return sentence
+
+
+# Try to install flash-attn if not available
+try:
+    import flash_attn
+    use_flash_attn = True
+except ImportError:
+    print("Flash attention not available, using standard attention")
+    use_flash_attn = False
+    try:
+        subprocess.run(
+            "pip install websockets",
+            shell=True,
+            check=False
+        )
+        subprocess.run(
+            "pip install flash-attn --no-build-isolation",
+            shell=True,
+            check=False
+        )
+    except:
+        pass
+
+device = "cuda" if torch.cuda.is_available() else "cpu"
+# --- Facebook Denoiser added ---
+try:
+    import torchaudio
+    from denoiser import pretrained
+    # Load DNS64 pretrained model (auto-downloads if not cached)
+    denoiser_model = pretrained.dns64().to(device)
+    denoiser_model.eval()
+    denoiser_available = True
+    print("facebook/denoiser loaded successfully")
+except ImportError as e:
+    denoiser_available = False
+    print("facebook/denoiser not available - install with: pip install denoiser torchaudio")
+    denoiser_model = None
+
+
+torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+MODEL_NAME = "alaatiger989/FT_Arabic_Whisper_V1_1"#"openai/whisper-large-v3-turbo"
+
+print(f"Using device: {device}")
+print(f"CUDA available: {torch.cuda.is_available()}")
+MODEL_NAME = "speechbrain/asr-whisper-large-v2-commonvoice-ar"
+# Replace your pipeline definition
+from speechbrain.inference.ASR import WhisperASR
+
+# Load the SpeechBrain model
+model = WhisperASR.from_hparams(
+    source="speechbrain/asr-whisper-large-v2-commonvoice-ar",
+    savedir="pretrained_models/asr-whisper-large-v2-commonvoice-ar",
+    run_opts={"device": "cuda"} if torch.cuda.is_available() else {}
+)
+
+
+def denoise_audio(audio_data, sample_rate=16000):
+    """Apply denoising using facebook/denoiser pretrained model."""
+    if denoiser_model is None or len(audio_data) == 0:
+        return audio_data
+    try:
+        audio_tensor = torch.tensor(audio_data, dtype=torch.float32, device=device).unsqueeze(0)
+        with torch.no_grad():
+            denoised_tensor = denoiser_model(audio_tensor)[0]  # no sample_rate arg
+        return denoised_tensor.squeeze().cpu().numpy().astype("float32")
+    except Exception as e:
+        print(f"[WARN] Denoiser failed: {e}")
+        return audio_data
+
+# Thread pool for processing audio
+executor = ThreadPoolExecutor(max_workers=4)
+
+class JambonzAudioBuffer:
+    def __init__(self, sample_rate=8000, chunk_duration=1.0):
+        self.sample_rate = sample_rate
+        self.chunk_duration = chunk_duration
+        self.chunk_samples = int(chunk_duration * sample_rate)
+        
+        self.buffer = np.array([], dtype=np.float32)
+        self.lock = threading.Lock()
+        self.total_audio = np.array([], dtype=np.float32)
+        
+        # Voice Activity Detection (simple energy-based)
+        self.silence_threshold = 0.01
+        self.min_speech_samples = int(0.3 * sample_rate)  # 300ms minimum speech
+        
+    def add_audio(self, audio_data):
+        with self.lock:
+            self.buffer = np.concatenate([self.buffer, audio_data])
+            self.total_audio = np.concatenate([self.total_audio, audio_data])
+    
+    def has_chunk_ready(self):
+        with self.lock:
+            return len(self.buffer) >= self.chunk_samples
+    
+    def is_speech(self, audio_chunk):
+        """Simple VAD based on energy"""
+        if len(audio_chunk) < self.min_speech_samples:
+            return False
+        energy = np.mean(np.abs(audio_chunk))
+        return energy > self.silence_threshold
+    
+    def get_chunk_for_processing(self):
+        """Get audio chunk for processing - but don't remove it from buffer for interim results"""
+        with self.lock:
+            if len(self.buffer) < self.chunk_samples:
+                return None
+            
+            # For interim results, we want to trigger processing but keep accumulating audio
+            # So we just return a signal that we have enough audio, but don't consume it
+            return np.array([1])  # Return a dummy array to signal chunk is ready
+    
+    def get_all_audio(self):
+        """Get all accumulated audio for final transcription"""
+        with self.lock:
+            return self.total_audio.copy()
+    
+    def clear(self):
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+
+def linear16_to_audio(audio_bytes, sample_rate=8000):
+    """Convert LINEAR16 PCM bytes to numpy array (jambonz format)"""
+    try:
+        # jambonz sends LINEAR16 PCM at 8kHz
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16)
+        # Convert to float32 and normalize
+        audio_array = audio_array.astype(np.float32) / 32768.0
+        return audio_array
+    except Exception as e:
+        logger.error(f"Error converting LINEAR16 to audio: {e}")
+        return np.array([], dtype=np.float32)
+
+def resample_audio(audio_data, source_rate, target_rate):
+    """Simple resampling from 8kHz to 16kHz"""
+    if source_rate == target_rate:
+        return audio_data
+    ratio = target_rate / source_rate
+    indices = np.arange(0, len(audio_data), 1/ratio)
+    indices = indices[indices < len(audio_data)]
+    resampled = np.interp(indices, np.arange(len(audio_data)), audio_data)
+    return resampled.astype(np.float32)
+
+
+import os
+import tempfile
+import soundfile as sf
+import logging
+
+logger = logging.getLogger(__name__)
+from pathlib import Path
+import uuid
+import shutil
+# Project-level temp folder
+PROJECT_DIR = Path(__file__).parent.resolve()
+AUDIO_TMP_DIR = PROJECT_DIR / "temp_audio"
+AUDIO_TMP_DIR.mkdir(exist_ok=True)
+def transcribe_chunk_direct(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    try:
+        if len(audio_data) == 0:
+            return ""
+
+        # Step 1: Resample
+        resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+
+        # Step 2: Denoise
+        resampled_audio = denoise_audio(resampled_audio)
+
+        # Step 3: Check minimum length (100ms)
+        min_samples = int(0.1 * target_sample_rate)
+        if len(resampled_audio) < min_samples:
+            return ""
+
+        # Step 4: Convert numpy -> torch tensor
+        waveform = torch.tensor(resampled_audio, dtype=torch.float32).unsqueeze(0)  # [1, T]
+
+        # Step 5: Create wav_lens (normalized length)
+        wav_lens = torch.tensor([1.0])  # full length, no padding
+
+        # Step 6: Transcribe
+        words, tokens = model.transcribe_batch(waveform, wav_lens)
+
+        # Step 7: Convert list of words to a sentence
+        transcription = " ".join(words[0]) if words and len(words) > 0 else ""
+
+        logger.info(f"SpeechBrain transcription: '{transcription}'")
+        return transcription.strip()
+
+    except Exception as e:
+        logger.error(f"Error during SpeechBrain transcription: {e}")
+        return ""
+
+
+# def transcribe_chunk_direct(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+#     """Transcribe audio chunk using model's generate method directly"""
+#     try:
+#         if len(audio_data) == 0:
+#             return ""
+        
+#         # Resample from 8kHz to 16kHz for Whisper
+#         resampled_audio = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        
+#         # Ensure minimum length for Whisper
+#         min_samples = int(0.1 * target_sample_rate)  # 100ms minimum
+#         if len(resampled_audio) < min_samples:
+#             return ""
+        
+#         start_time = time.time()
+        
+#         # Prepare input features with proper dtype
+#         input_features = processor(
+#             resampled_audio, 
+#             sampling_rate=target_sample_rate, 
+#             return_tensors="pt"
+#         ).input_features
+        
+#         # Ensure correct dtype and device
+#         input_features = input_features.to(device=device, dtype=torch_dtype)
+        
+#         # Create attention mask to avoid warnings
+#         attention_mask = torch.ones(
+#             input_features.shape[:-1], 
+#             dtype=torch.long, 
+#             device=device
+#         )
+        
+#         # Generate transcription using model directly
+#         with torch.no_grad():
+#             predicted_ids = model.generate(
+#                 input_features,
+#                 attention_mask=attention_mask,
+#                 max_new_tokens=128,
+#                 do_sample=False,
+#                 temperature=0.0,
+#                 num_beams=1,
+#                 language="ar",
+#                 task="transcribe",
+#                 pad_token_id=tokenizer.pad_token_id,
+#                 eos_token_id=tokenizer.eos_token_id
+#             )
+        
+#         # Decode the transcription
+#         transcription = tokenizer.batch_decode(
+#             predicted_ids, 
+#             skip_special_tokens=True
+#         )[0].strip()
+        
+#         end_time = time.time()
+        
+#         logger.info(f"Direct transcription completed in {end_time - start_time:.2f}s: '{transcription}'")
+#         return transcription
+        
+#     except Exception as e:
+#         logger.error(f"Error during direct transcription: {e}")
+#         return ""
+
+class JambonzSTTHandler:
+    def __init__(self, websocket):
+        self.websocket = websocket
+        self.audio_buffer = None
+        self.config = {}
+        self.running = True
+        self.transcription_task = None
+        self.full_transcript = ""
+        self.last_partial = ""
+        
+        # Auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.silence_timeout = 1.5  # 3 seconds of silence to trigger final
+        self.min_interim_count = 1  # Minimum interim results before considering final
+        self.auto_final_task = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        
+    async def start_processing(self, start_message):
+        """Initialize with start message from jambonz"""
+        self.config = {
+            "language": start_message.get("language", "ar-EG"),
+            "format": start_message.get("format", "raw"),
+            "encoding": start_message.get("encoding", "LINEAR16"),
+            "sample_rate": start_message.get("sampleRateHz", 8000),
+            "interim_results": start_message.get("interimResults", True),
+            "options": start_message.get("options", {})
+        }
+        
+        logger.info(f"STT session started with config: {self.config}")
+        
+        # Initialize audio buffer
+        self.audio_buffer = JambonzAudioBuffer(
+            sample_rate=self.config["sample_rate"],
+            chunk_duration=1.0  # Process every 1 second
+        )
+        
+        # Reset auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        
+        # Start background transcription task
+        self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+        # Start auto-final detection task
+        self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+    
+    async def stop_processing(self):
+        """Stop processing and send final transcription"""
+        self.running = False
+        
+        # Cancel background tasks
+        if self.transcription_task:
+            self.transcription_task.cancel()
+            try:
+                await self.transcription_task
+            except asyncio.CancelledError:
+                pass
+        
+        if self.auto_final_task:
+            self.auto_final_task.cancel()
+            try:
+                await self.auto_final_task
+            except asyncio.CancelledError:
+                pass
+        
+        # Send final transcription if not already sent
+        if not self.final_sent and self.accumulated_transcript.strip():
+            await self.send_transcription(self.accumulated_transcript, is_final=True)
+        
+        # Also process any remaining audio for comprehensive final transcription
+        if self.audio_buffer:
+            all_audio = self.audio_buffer.get_all_audio()
+            if len(all_audio) > 0 and not self.final_sent:
+                loop = asyncio.get_event_loop()
+                final_transcription = await loop.run_in_executor(
+                    executor, 
+                    transcribe_chunk_direct, 
+                    all_audio,
+                    self.config["sample_rate"]
+                )
+                
+                if final_transcription.strip():
+                    # Send comprehensive final transcription
+                    await self.send_transcription(final_transcription, is_final=True)
+        
+        logger.info("STT session ended")
+    
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer"""
+        if self.audio_buffer:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+    
+    async def _process_audio_chunks(self):
+        """Process audio chunks for interim results"""
+        while self.running and self.config.get("interim_results", False):
+            try:
+                if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+                    chunk_signal = self.audio_buffer.get_chunk_for_processing()
+                    if chunk_signal is not None:
+                        # Get all accumulated audio so far for complete transcription
+                        all_audio = self.audio_buffer.get_all_audio()
+                        
+                        # Only process if we have actual speech content
+                        if len(all_audio) > 0 and self.audio_buffer.is_speech(all_audio[-self.audio_buffer.chunk_samples:]):
+                            # Run transcription on all accumulated audio
+                            loop = asyncio.get_event_loop()
+                            transcription = await loop.run_in_executor(
+                                executor, 
+                                transcribe_chunk_direct, 
+                                all_audio,
+                                self.config["sample_rate"]
+                            )
+                            
+                            if transcription.strip() and transcription != self.last_partial:
+                                self.last_partial = transcription
+                                self.accumulated_transcript = transcription  # Update accumulated transcript
+                                self.interim_count += 1
+                                self.last_interim_time = time.time()
+                                
+                                # Send interim result
+                                await self.send_transcription(transcription, is_final=False)
+                                
+                                logger.info(f"Interim #{self.interim_count}: '{transcription}'")
+                
+                # Small delay to prevent excessive processing
+                await asyncio.sleep(0.1)
+                
+            except Exception as e:
+                logger.error(f"Error in chunk processing: {e}")
+                await asyncio.sleep(1)
+    
+    async def _monitor_for_auto_final(self):
+        """Monitor for auto-final conditions: 3 seconds silence after 3+ interim results"""
+        while self.running:
+            try:
+                current_time = time.time()
+                
+                # Check if we should send auto-final transcription
+                if (self.interim_count >= self.min_interim_count and 
+                    self.last_interim_time is not None and 
+                    (current_time - self.last_interim_time) >= self.silence_timeout and
+                    not self.final_sent and
+                    self.accumulated_transcript.strip()):
+                    
+                    logger.info(f"Auto-final triggered: {self.interim_count} interim results, "
+                              f"{current_time - self.last_interim_time:.1f}s silence")
+                    
+                    # Send the accumulated transcript as final
+                    await self.send_transcription(self.accumulated_transcript, is_final=True)
+                    self.final_sent = True
+                    
+                    # Reset counters for potential next utterance
+                    self.interim_count = 0
+                    self.last_interim_time = None
+                    self.accumulated_transcript = ""
+                
+                # Check every 0.5 seconds
+                await asyncio.sleep(0.5)
+                
+            except Exception as e:
+                logger.error(f"Error in auto-final monitoring: {e}")
+                await asyncio.sleep(1)
+    
+    # async def send_transcription(self, text, is_final=False, confidence=0.9):
+    #     """Send transcription in jambonz format with Arabic number conversion"""
+    #     try:
+    #         # Convert Arabic numbers to digits before sending
+    #         original_text = text
+    #         converted_text = convert_arabic_numbers_in_sentence(text)
+            
+    #         # Log the conversion if numbers were found and converted
+    #         if original_text != converted_text:
+    #             logger.info(f"Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+            
+    #         message = {
+    #             "type": "transcription",
+    #             "is_final": is_final,
+    #             "alternatives": [
+    #                 {
+    #                     "transcript": converted_text,
+    #                     "confidence": confidence
+    #                 }
+    #             ],
+    #             "language": self.config.get("language", "ar-EG"),
+    #             "channel": 1
+    #         }
+            
+    #         await self.websocket.send(json.dumps(message))
+    #         logger.info(f"Sent {'FINAL' if is_final else 'interim'} transcription: '{converted_text}'")
+            
+    #         if is_final:
+    #             self.final_sent = True
+            
+    #     except Exception as e:
+    #         logger.error(f"Error sending transcription: {e}")
+    
+
+
+    async def send_transcription(self, text, is_final=False, confidence=0.9):
+        """Send transcription in jambonz format with Arabic number conversion, only for final results"""
+        try:
+            if not is_final:
+                # Do nothing for interim results
+                logger.debug("Skipping interim transcription (not final).")
+                return
+
+            # Convert Arabic numbers only for final transcripts
+            original_text = text
+            converted_text = convert_arabic_numbers_in_sentence(text)
+
+            # Log the conversion if numbers were found and converted
+            if original_text != converted_text:
+                logger.info(f"Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+
+            message = {
+                "type": "transcription",
+                "is_final": True,
+                "alternatives": [
+                    {
+                        "transcript": original_text,#converted_text,
+                        "confidence": confidence
+                    }
+                ],
+                "language": self.config.get("language", "ar-EG"),
+                "channel": 1
+            }
+
+            # Send only final messages
+            await self.websocket.send(json.dumps(message))
+            logger.info(f"Sent FINAL transcription: '{converted_text}'")
+
+            self.final_sent = True
+
+        except Exception as e:
+            logger.error(f"Error sending transcription: {e}")
+
+
+
+
+    async def send_error(self, error_message):
+        """Send error message in jambonz format"""
+        try:
+            message = {
+                "type": "error",
+                "error": error_message
+            }
+            await self.websocket.send(json.dumps(message))
+            logger.error(f"Sent error: {error_message}")
+        except Exception as e:
+            logger.error(f"Error sending error message: {e}")
+
+async def handle_jambonz_websocket(websocket):
+    """Handle jambonz WebSocket connections"""
+    
+    client_id = f"jambonz_{id(websocket)}"
+    logger.info(f"New jambonz connection: {client_id}")
+    
+    handler = JambonzSTTHandler(websocket)
+    
+    try:
+        async for message in websocket:
+            try:
+                if isinstance(message, str):
+                    # Handle JSON control messages
+                    data = json.loads(message)
+                    message_type = data.get("type")
+                    
+                    if message_type == "start":
+                        logger.info(f"Received start message: {data}")
+                        await handler.start_processing(data)
+                        
+                    elif message_type == "stop":
+                        logger.info("Received stop message")
+                        await handler.stop_processing()
+                        # Close websocket after final transcription
+                        await websocket.close(code=1000, reason="Session completed")
+                        break
+                        
+                    else:
+                        logger.warning(f"Unknown message type: {message_type}")
+                        await handler.send_error(f"Unknown message type: {message_type}")
+                
+                else:
+                    # Handle binary audio data (LINEAR16 PCM)
+                    if handler.audio_buffer is None:
+                        await handler.send_error("Received audio before start message")
+                        continue
+                    
+                    await handler.add_audio_data(message)
+            
+            except json.JSONDecodeError as e:
+                logger.error(f"JSON decode error: {e}")
+                await handler.send_error(f"Invalid JSON: {str(e)}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+                await handler.send_error(f"Processing error: {str(e)}")
+    
+    except websockets.exceptions.ConnectionClosed:
+        logger.info(f"jambonz connection closed: {client_id}")
+    except Exception as e:
+        logger.error(f"jambonz WebSocket error: {e}")
+        try:
+            await handler.send_error(str(e))
+        except:
+            pass
+    finally:
+        if handler.running:
+            await handler.stop_processing()
+        logger.info(f"jambonz connection ended: {client_id}")
+
+async def main():
+    """Start the jambonz STT WebSocket server"""
+    logger.info("Starting Jambonz Custom STT WebSocket server on port 3006...")
+    
+    # Start WebSocket server
+    server = await websockets.serve(
+        handle_jambonz_websocket,
+        "0.0.0.0", 
+        3006,
+        ping_interval=20,
+        ping_timeout=10,
+        close_timeout=10
+    )
+    
+    logger.info("Jambonz Custom STT WebSocket server started on ws://0.0.0.0:3006")
+    logger.info("Ready to handle jambonz STT requests")
+    logger.info("- Expects LINEAR16 PCM audio at 8kHz")
+    logger.info("- Supports interim results with auto-final detection")
+    logger.info("- Auto-final: 3+ interim results + 1.3s silence")
+    logger.info("- Resamples to 16kHz for Whisper processing")
+    logger.info("- Converts Arabic numbers to digits before sending")
+    
+    # Wait for the server to close
+    await server.wait_closed()
+
+if __name__ == "__main__":
+    print("=" * 60)
+    print("Jambonz Custom STT Server with Whisper + Arabic Numbers")
+    print("=" * 60)
+    print(f"Model: {MODEL_NAME}")
+    print(f"Device: {device}")
+    print("WebSocket Port: 3006")
+    print("Protocol: jambonz STT API")
+    print("Audio Format: LINEAR16 PCM @ 8kHz")
+    print("Auto-Final: 2+ speech activities + 1.3s silence")
+    print("Arabic Numbers: Converted to digits in FINAL transcriptions only")
+    print("Interim Results: DISABLED (final transcription only)")
+    if arabic_numbers_available:
+        print("✓ pyarabic library available for number conversion")
+    else:
+        print("✗ pyarabic library not available - install with: pip install pyarabic")
+    print("=" * 60)
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nShutting down server...")
+    except Exception as e:
+        print(f"Server error: {e}")

stt_ar_fastconformer_hybrid_large_pcd_v1.0.nemo

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d29d19d7c054a5fc010ac6815e9cbb0dd1b21a30e0a7f7f2982e1fecaf0c3e31
+size 459233280

w_nemo.py

@@ -0,0 +1,1033 @@
+import torch
+import asyncio
+import websockets
+import json
+import threading
+import numpy as np
+import logging
+import time
+import tempfile
+import os
+import re
+from concurrent.futures import ThreadPoolExecutor
+import subprocess
+import struct
+
+# NeMo imports
+import nemo.collections.asr as nemo_asr
+import soundfile as sf
+
+# Whisper imports
+# from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, WhisperTokenizer, pipeline
+from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
+
+
+# Arabic number conversion imports for Whisper
+try:
+    from pyarabic.number import text2number
+    arabic_numbers_available = True
+    print("✓ pyarabic library available for Whisper number conversion")
+except ImportError:
+    arabic_numbers_available = False
+    print("✗ pyarabic not available - install with: pip install pyarabic")
+    print("Arabic numbers will not be converted to digits for Whisper")
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# ===== NeMo Arabic number mapping =====
+arabic_numbers_nemo = {
+    # Basic digits
+    "سفر": "0", "فيرو": "0", "هيرو": "0","صفر": "0", "زيرو": "0", "٠": "0","زيو": "0","زير": "0","زير": "0","زر": "0","زروا": "0","زرا": "0","زيره ": "0","زرو ": "0",
+    "واحد": "1", "واحدة": "1", "١": "1",
+    "اتنين": "2", "اثنين": "2", "إثنين": "2", "اثنان": "2", "إثنان": "2", "٢": "2",
+    "تلاتة": "3", "ثلاثة": "3", "٣": "3","تلاته": "3","ثلاثه": "3","ثلاثا": "3","تلاتا": "3",
+    "اربعة": "4", "أربعة": "4", "٤": "4","اربعه": "4","أربعه": "4","أربع": "4","اربع": "4","اربعا": "4","أربعا": "4",
+    "خمسة": "5", "خمسه": "5", "٥": "5", "خمس": "5", "خمسا": "5",
+    "ستة": "6", "سته": "6", "٦": "6", "ست": "6", "ستّا": "6", "ستةً": "6",
+    "سبعة": "7", "سبعه": "7", "٧": "7", "سبع": "7", "سبعا": "7",
+    "ثمانية": "8", "ثمانيه": "8", "٨": "8", "ثمان": "8", "ثمنية": "8", "ثمنيه": "8", "ثمانيا": "8", "ثمن": "8",
+    "تسعة": "9", "تسعه": "9", "٩": "9", "تسع": "9", "تسعا": "9",
+    
+    # Teens
+    "عشرة": "10", "١٠": "10",
+    "حداشر": "11", "احد عشر": "11","احداشر": "11",
+    "اتناشر": "12", "اثنا عشر": "12",
+    "تلتاشر": "13", "ثلاثة عشر": "13",
+    "اربعتاشر": "14", "أربعة عشر": "14",
+    "خمستاشر": "15", "خمسة عشر": "15",
+    "ستاشر": "16", "ستة عشر": "16",
+    "سبعتاشر": "17", "سبعة عشر": "17",
+    "طمنتاشر": "18", "ثمانية عشر": "18",
+    "تسعتاشر": "19", "تسعة عشر": "19",
+    
+    # Tens
+    "عشرين": "20", "٢٠": "20",
+    "تلاتين": "30", "ثلاثين": "30", "٣٠": "30",
+    "اربعين": "40", "أربعين": "40", "٤٠": "40",
+    "خمسين": "50", "٥٠": "50",
+    "ستين": "60", "٦٠": "60",
+    "سبعين": "70", "٧٠": "70",
+    "تمانين": "80", "ثمانين": "80", "٨٠": "80","تمانون": "80","ثمانون": "80",
+    "تسعين": "90", "٩٠": "90",
+    
+    # Hundreds
+    "مية": "100", "مائة": "100", "مئة": "100", "١٠٠": "100",
+    "ميتين": "200", "مائتين": "200",
+    "تلاتمية": "300", "ثلاثمائة": "300",
+    "اربعمية": "400", "أربعمائة": "400",
+    "خمسمية": "500", "خمسمائة": "500",
+    "ستمية": "600", "ستمائة": "600",
+    "سبعمية": "700", "سبعمائة": "700",
+    "تمانمية": "800", "ثمانمائة": "800",
+    "تسعمية": "900", "تسعمائة": "900",
+    
+    # Thousands
+    "ألف": "1000", "الف": "1000", "١٠٠٠": "1000",
+    "ألفين": "2000", "الفين": "2000",
+    "تلات تلاف": "3000", "ثلاثة آلاف": "3000",
+    "اربعة آلاف": "4000", "أربعة آلاف": "4000",
+    "خمسة آلاف": "5000",
+    "ستة آلاف": "6000",
+    "سبعة آلاف": "7000",
+    "تمانية آلاف": "8000", "ثمانية آلاف": "8000",
+    "تسعة آلاف": "9000",
+    
+    # Large numbers
+    "عشرة آلاف": "10000",
+    "مية ألف": "100000", "مائة ألف": "100000",
+    "مليون": "1000000", "١٠٠٠٠٠٠": "1000000",
+    "ملايين": "1000000",
+    "مليار": "1000000000", "١٠٠٠٠٠٠٠٠٠": "1000000000"
+}
+
+def replace_arabic_numbers_nemo(text: str) -> str:
+    """Convert Arabic number words to digits for NeMo"""
+    for word, digit in arabic_numbers_nemo.items():
+        text = re.sub(rf"\b{word}\b", digit, text)
+    return text
+
+def convert_arabic_numbers_whisper(sentence: str) -> str:
+    """
+    Replace Arabic number words in a sentence with digits for Whisper,
+    preserving all other words and punctuation.
+    """
+    if not arabic_numbers_available or not sentence.strip():
+        return sentence
+    
+    try:
+        # Normalization step
+        replacements = {
+            "اربعة": "أربعة", "اربع": "أربع", "اثنين": "اثنان",
+            "اتنين": "اثنان", "ثلاث": "ثلاثة", "خمس": "خمسة",
+            "ست": "ستة", "سبع": "سبعة", "ثمان": "ثمانية",
+            "تسع": "تسعة", "عشر": "عشرة",
+        }
+        for wrong, correct in replacements.items():
+            sentence = re.sub(rf"\b{wrong}\b", correct, sentence)
+
+        # Split by whitespace but keep spaces
+        words = re.split(r'(\s+)', sentence)
+        converted_words = []
+
+        for word in words:
+            stripped = word.strip()
+            if not stripped:  # skip spaces
+                converted_words.append(word)
+                continue
+
+            try:
+                num = text2number(stripped)
+                if isinstance(num, int):
+                    if num != 0 or stripped == "صفر":
+                        converted_words.append(str(num))
+                    else:
+                        converted_words.append(word)
+                else:
+                    converted_words.append(word)
+            except Exception:
+                converted_words.append(word)
+
+        return ''.join(converted_words)
+
+    except Exception as e:
+        logger.warning(f"Error converting Arabic numbers: {e}")
+        return sentence
+
+# Global models
+asr_model_nemo = None
+whisper_model = None
+whisper_processor = None
+whisper_tokenizer = None
+device = None
+torch_dtype = None
+
+def initialize_models():
+    """Initialize both NeMo and Whisper models"""
+    global asr_model_nemo, whisper_model, whisper_processor, whisper_tokenizer, device, torch_dtype
+    
+    # Initialize device settings
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
+    
+    logger.info(f"Using device: {device}")
+    logger.info(f"CUDA available: {torch.cuda.is_available()}")
+    
+    # Initialize NeMo model
+    logger.info("Loading NeMo FastConformer Arabic ASR model...")
+    model_path = "stt_ar_fastconformer_hybrid_large_pcd_v1.0.nemo"
+    
+    if os.path.exists(model_path):
+        try:
+            asr_model_nemo = nemo_asr.models.EncDecCTCModel.restore_from(model_path)
+            asr_model_nemo.eval()
+            logger.info("✓ NeMo FastConformer model loaded successfully")
+        except Exception as e:
+            logger.error(f"Failed to load NeMo model: {e}")
+            asr_model_nemo = None
+    else:
+        logger.warning(f"NeMo model not found at: {model_path}")
+        asr_model_nemo = None
+    
+    # Initialize Whisper model
+    # from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor
+
+    logger.info("Loading Whisper large-v3 model...")
+    MODEL_NAME = "alaatiger989/FT_Arabic_Whisper_V1_1"
+
+    try:
+        # Try with flash attention first
+        try:
+            import flash_attn
+            whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+                MODEL_NAME,
+                torch_dtype=torch_dtype,
+                low_cpu_mem_usage=True,
+                use_safetensors=True,
+                attn_implementation="flash_attention_2"
+            )
+            logger.info("✓ Whisper loaded with flash attention")
+        except:
+            whisper_model = AutoModelForSpeechSeq2Seq.from_pretrained(
+                MODEL_NAME,
+                torch_dtype=torch_dtype,
+                low_cpu_mem_usage=True,
+                use_safetensors=True
+            )
+            logger.info("✓ Whisper loaded with standard attention")
+
+        whisper_model.to(device)
+        whisper_processor = AutoProcessor.from_pretrained(MODEL_NAME)
+
+        # Use processor.tokenizer, don’t reload separately
+        whisper_tokenizer = whisper_processor.tokenizer  
+
+        logger.info("✓ Whisper model + tokenizer loaded successfully")
+
+    except Exception as e:
+        logger.error(f"Failed to load Whisper model: {e}")
+        whisper_model = None
+
+# Initialize models on startup
+initialize_models()
+
+# Thread pool for processing
+executor = ThreadPoolExecutor(max_workers=4)
+
+
+
+class JambonzAudioBuffer:
+    def __init__(self, sample_rate=8000, chunk_duration=1.0):
+        self.sample_rate = sample_rate
+        self.chunk_duration = chunk_duration
+        self.chunk_samples = int(chunk_duration * sample_rate)
+        
+        self.buffer = np.array([], dtype=np.float32)
+        self.lock = threading.Lock()
+        self.total_audio = np.array([], dtype=np.float32)
+        
+        # Voice Activity Detection - ADJUSTED FOR WHISPER
+        self.silence_threshold = 0.01  # Lower threshold for Whisper
+        self.min_speech_samples = int(0.3 * sample_rate)  # 300ms minimum speech
+        
+    def add_audio(self, audio_data):
+        with self.lock:
+            self.buffer = np.concatenate([self.buffer, audio_data])
+            self.total_audio = np.concatenate([self.total_audio, audio_data])
+            
+            # Log audio addition for debugging
+            logger.debug(f"Added {len(audio_data)} audio samples, total: {len(self.total_audio)}")
+    
+    def has_chunk_ready(self):
+        with self.lock:
+            ready = len(self.buffer) >= self.chunk_samples
+            if ready:
+                logger.debug(f"Chunk ready: {len(self.buffer)} >= {self.chunk_samples}")
+            return ready
+    
+    def is_speech(self, audio_chunk):
+        """Enhanced VAD based on energy - better for Whisper"""
+        if len(audio_chunk) < self.min_speech_samples:
+            logger.debug(f"Audio too short for VAD: {len(audio_chunk)} < {self.min_speech_samples}")
+            return False
+            
+        # Calculate RMS energy
+        rms_energy = np.sqrt(np.mean(audio_chunk ** 2))
+        
+        # Also check peak amplitude
+        peak_amplitude = np.max(np.abs(audio_chunk))
+        
+        is_speech = rms_energy > self.silence_threshold or peak_amplitude > (self.silence_threshold * 2)
+        
+        logger.debug(f"VAD check - RMS: {rms_energy:.4f}, Peak: {peak_amplitude:.4f}, "
+                    f"Threshold: {self.silence_threshold}, Speech: {is_speech}")
+        
+        return is_speech
+    
+    def get_chunk_for_processing(self):
+        """Get audio chunk for processing"""
+        with self.lock:
+            if len(self.buffer) < self.chunk_samples:
+                return None
+            
+            logger.debug(f"Returning processing signal, buffer size: {len(self.buffer)}")
+            return np.array([1])  # Signal that chunk is ready
+    
+    def get_all_audio(self):
+        """Get all accumulated audio"""
+        with self.lock:
+            audio_copy = self.total_audio.copy()
+            logger.debug(f"Returning {len(audio_copy)} total audio samples")
+            return audio_copy
+    
+    def clear(self):
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+            logger.debug("Audio buffer cleared")
+    
+    def reset_for_new_segment(self):
+        """Reset buffers for new transcription segment"""
+        with self.lock:
+            self.buffer = np.array([], dtype=np.float32)
+            self.total_audio = np.array([], dtype=np.float32)
+            logger.debug("Audio buffer reset for new segment")
+
+def linear16_to_audio(audio_bytes, sample_rate=8000):
+    """Convert LINEAR16 PCM bytes to numpy array"""
+    try:
+        audio_array = np.frombuffer(audio_bytes, dtype=np.int16)
+        audio_array = audio_array.astype(np.float32) / 32768.0
+        return audio_array
+    except Exception as e:
+        logger.error(f"Error converting LINEAR16 to audio: {e}")
+        return np.array([], dtype=np.float32)
+
+from scipy.signal import resample_poly
+
+# def resample_audio(audio_data, source_rate, target_rate):
+#     """High-quality resampling using polyphase resampler."""
+#     if source_rate == target_rate:
+#         return audio_data.astype(np.float32)
+#     # convert float32 [-1..1] to float32 still, but resample
+#     gcd = np.gcd(source_rate, target_rate)
+#     up = target_rate // gcd
+#     down = source_rate // gcd
+#     # resample_poly expects 1D numpy array
+#     try:
+#         resampled = resample_poly(audio_data, up, down).astype(np.float32)
+#         return resampled
+#     except Exception as e:
+#         logger.warning(f"resample_audio fallback: {e}")
+#         # last-resort simple repeat (keep previous behavior) but warn
+#         if source_rate == 8000 and target_rate == 16000:
+#             return np.repeat(audio_data, 2).astype(np.float32)
+#         return audio_data.astype(np.float32)
+
+import numpy as np
+from scipy.signal import resample_poly, butter, lfilter
+import webrtcvad
+import noisereduce as nr
+
+# Initialize WebRTC VAD once (0..3, higher = more aggressive/noisy environments)
+_vad = webrtcvad.Vad(2)
+
+def resample_audio(audio_data, source_rate, target_rate=16000,
+                              lowcut=80.0, highcut=7600.0,
+                              frame_ms=30, required_ratio=0.55):
+    """
+    Resample -> Bandpass filter -> Noise reduction -> WebRTC VAD speech detection.
+
+    Returns:
+        processed_audio (np.ndarray float32): cleaned/resampled audio
+        is_speech (bool): True if VAD detects speech
+    """
+
+    # --- Resample ---
+    if source_rate != target_rate:
+        gcd = np.gcd(source_rate, target_rate)
+        up = target_rate // gcd
+        down = source_rate // gcd
+        try:
+            audio_data = resample_poly(audio_data, up, down).astype(np.float32)
+        except Exception:
+            audio_data = np.repeat(audio_data, int(target_rate/source_rate)).astype(np.float32)
+    else:
+        audio_data = audio_data.astype(np.float32)
+
+    # --- Bandpass filter (speech range) ---
+    try:
+        nyq = 0.5 * target_rate
+        low = lowcut / nyq
+        high = highcut / nyq
+        b, a = butter(4, [low, high], btype='band')
+        audio_data = lfilter(b, a, audio_data).astype(np.float32)
+    except Exception:
+        pass
+
+    # --- Noise reduction ---
+    try:
+        if len(audio_data) >= int(0.25 * target_rate):
+            noise_clip = audio_data[:int(0.25 * target_rate)]
+            audio_data = nr.reduce_noise(y=audio_data, y_noise=noise_clip, sr=target_rate).astype(np.float32)
+    except Exception:
+        pass
+
+    # --- WebRTC VAD ---
+    def frame_generator(frame_ms, audio, sample_rate):
+        n = int(sample_rate * (frame_ms / 1000.0))
+        if len(audio) < n:
+            return
+        offset = 0
+        while offset + n <= len(audio):
+            frame = audio[offset:offset+n]
+            yield (frame * 32767).astype(np.int16).tobytes()
+            offset += n
+
+    frames = list(frame_generator(frame_ms, audio_data, target_rate))
+    voiced = 0
+    for f in frames:
+        try:
+            if _vad.is_speech(f, target_rate):
+                voiced += 1
+        except Exception:
+            pass
+    ratio = voiced / max(1, len(frames))
+    is_speech = ratio >= required_ratio
+
+    return audio_data, is_speech
+
+def transcribe_with_nemo(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio using NeMo FastConformer"""
+    try:
+        if len(audio_data) == 0 or asr_model_nemo is None:
+            return ""
+        
+        # Resample to 16kHz (NeMo models typically expect 16kHz)
+        resampled_audio, has_speech = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        
+        if has_speech:
+            print("Speech detected, sending to ASR...")
+            # Skip very short audio
+            min_samples = int(0.3 * target_sample_rate)
+            if len(resampled_audio) < min_samples:
+                return ""
+            
+            start_time = time.time()
+            
+            # Save audio to temporary file (NeMo expects file path)
+            with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
+                sf.write(tmp_file.name, resampled_audio, target_sample_rate)
+                tmp_path = tmp_file.name
+            
+            try:
+                # Transcribe with NeMo
+                result = asr_model_nemo.transcribe([tmp_path])
+                
+                if result and len(result) > 0:
+                    # Handle different NeMo result formats
+                    if hasattr(result[0], 'text'):
+                        raw_text = result[0].text
+                    elif isinstance(result[0], str):
+                        raw_text = result[0]
+                    else:
+                        raw_text = str(result[0])
+                    
+                    if not isinstance(raw_text, str):
+                        raw_text = str(raw_text)
+                    
+                    if raw_text and raw_text.strip():
+                        # Convert Arabic numbers to digits for NeMo
+                        cleaned_text = replace_arabic_numbers_nemo(raw_text)
+                        end_time = time.time()
+                        
+                        if cleaned_text.strip():
+                            logger.info(f"NeMo transcription: '{cleaned_text}' (processed in {end_time - start_time:.2f}s)")
+                        
+                        return cleaned_text.strip()
+                        
+            finally:
+                # Clean up temporary file
+                if os.path.exists(tmp_path):
+                    os.remove(tmp_path)
+            
+            return ""
+        else:
+            print("Silence/noise, skipping...")
+        
+    except Exception as e:
+        logger.error(f"Error during NeMo transcription: {e}")
+        return ""
+
+def transcribe_with_whisper(audio_data, source_sample_rate=8000, target_sample_rate=16000):
+    """Transcribe audio chunk using Whisper model directly"""
+    try:
+        if len(audio_data) == 0 or whisper_model is None:
+            return ""
+        
+        # Resample from 8kHz to 16kHz for Whisper
+        resampled_audio, has_speech = resample_audio(audio_data, source_sample_rate, target_sample_rate)
+        if has_speech:
+            print("Speech detected, sending to ASR...")
+            # Ensure minimum length for Whisper
+            min_samples = int(0.1 * target_sample_rate)  # 100ms minimum
+            if len(resampled_audio) < min_samples:
+                return ""
+            
+            start_time = time.time()
+            
+            # Prepare input features with proper dtype
+            input_features = whisper_processor(
+                resampled_audio, 
+                sampling_rate=target_sample_rate, 
+                return_tensors="pt"
+            ).input_features
+            
+            # Ensure correct dtype and device
+            input_features = input_features.to(device=device, dtype=torch_dtype)
+            
+            # Create attention mask to avoid warnings
+            attention_mask = torch.ones(
+                input_features.shape[:-1], 
+                dtype=torch.long, 
+                device=device
+            )
+            
+            # Generate transcription using model directly
+            with torch.no_grad():
+                predicted_ids = whisper_model.generate(
+                    input_features,
+                    attention_mask=attention_mask,
+                    max_new_tokens=128,
+                    do_sample=False,
+                    # temperature=0.0,
+                    num_beams=1,
+                    language="english",
+                    task="translate",
+                    pad_token_id=whisper_tokenizer.pad_token_id,
+                    eos_token_id=whisper_tokenizer.eos_token_id
+                )
+            
+            # Decode the transcription
+            transcription = whisper_tokenizer.batch_decode(
+                predicted_ids, 
+                skip_special_tokens=True
+            )[0].strip()
+            
+            end_time = time.time()
+            
+            logger.info(f"Whisper transcription completed in {end_time - start_time:.2f}s: '{transcription}'")
+            return transcription
+        else:
+            print("Silence/noise, skipping...")
+    except Exception as e:
+        logger.error(f"Error during Whisper transcription: {e}")
+        return ""
+
+class UnifiedSTTHandler:
+    def __init__(self, websocket):
+        self.websocket = websocket
+        self.audio_buffer = None
+        self.config = {}
+        self.running = False
+        self.transcription_task = None
+        self.use_nemo = False  # Flag to determine which model to use
+        
+        # Auto-final detection variables
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.silence_timeout = 2.9
+        self.min_interim_count = 1
+        self.auto_final_task = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.segment_number = 0
+        self.last_partial = ""
+        
+        # Processing tracking
+        self.processing_count = 0
+
+    # Add this debugging method to your UnifiedSTTHandler class
+
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer with enhanced debugging"""
+        if self.audio_buffer and self.running:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+            
+            model_name = "NeMo" if self.use_nemo else "Whisper"
+            
+            # Debug logging every few audio packets
+            if len(audio_data) > 0:
+                total_samples = len(self.audio_buffer.get_all_audio())
+                total_seconds = total_samples / self.config["sample_rate"]
+                
+                # Log every second of audio
+                if int(total_seconds) != getattr(self, '_last_logged_second', -1):
+                    logger.info(f"{model_name} - Accumulated {total_seconds:.1f}s of audio ({total_samples} samples)")
+                    self._last_logged_second = int(total_seconds)
+                    
+                    # Check if we should have chunks ready
+                    chunk_ready = self.audio_buffer.has_chunk_ready()
+                    logger.info(f"{model_name} - Chunk ready: {chunk_ready}")        
+
+    async def start_processing(self, start_message):
+        """Initialize with start message from jambonz"""
+        self.config = {
+            "language": start_message.get("language", "ar-EG"),
+            "format": start_message.get("format", "raw"),
+            "encoding": start_message.get("encoding", "LINEAR16"),
+            "sample_rate": start_message.get("sampleRateHz", 8000),
+            "interim_results": True,  # Always enable for internal processing
+            "options": start_message.get("options", {})
+        }
+        
+        # Determine which model to use based on language parameter
+        language = self.config["language"]
+        if language == "ar-EG":
+            logger.info("Selected NeMo FastConformer")
+            self.use_nemo = True
+            model_name = "NeMo FastConformer"
+        elif language == "ar-EG-whis":
+            logger.info("Selected Whisper large-v3")
+            self.use_nemo = False
+            model_name = "Whisper large-v3"
+        else:
+            # Default to NeMo for any other Arabic variant
+            self.use_nemo = True
+            model_name = "NeMo FastConformer (default)"
+        
+        logger.info(f"STT session started with {model_name} for language: {language}")
+        logger.info(f"Config: {self.config}")
+        
+        # Check if selected model is available
+        if self.use_nemo and asr_model_nemo is None:
+            await self.send_error("NeMo model not available")
+            return
+        elif not self.use_nemo and whisper_model is None:
+            await self.send_error("Whisper model not available")
+            return
+        
+        # Initialize audio buffer with model-specific settings
+        if self.use_nemo:
+            chunk_duration = 1.0  # NeMo processes every 1 second
+        else:
+            chunk_duration = 2.0  # Whisper processes every 2 seconds for better accuracy
+        
+        self.audio_buffer = JambonzAudioBuffer(
+            sample_rate=self.config["sample_rate"],
+            chunk_duration=chunk_duration
+        )
+        
+        # Adjust VAD threshold for Whisper
+        if not self.use_nemo:
+            self.audio_buffer.silence_threshold = 0.005  # Lower threshold for Whisper
+        
+        # Reset session variables
+        self.running = True
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.segment_number = 0
+        self.processing_count = 0
+        self.last_partial = ""
+        
+        # Start background transcription task
+        self.transcription_task = asyncio.create_task(self._process_audio_chunks())
+        
+        # Start auto-final detection task
+        self.auto_final_task = asyncio.create_task(self._monitor_for_auto_final())
+        
+        logger.info(f"Background tasks started for {model_name}")
+
+
+
+    async def stop_processing(self):
+        """Stop current processing session"""
+        logger.info("Stopping STT session...")
+        self.running = False
+        
+        # Cancel background tasks
+        for task in [self.transcription_task, self.auto_final_task]:
+            if task:
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+        
+        # Send final transcription if not already sent
+        if not self.final_sent and self.accumulated_transcript.strip():
+            await self.send_transcription(self.accumulated_transcript, is_final=True)
+        
+        # Process any remaining audio for comprehensive final transcription
+        if self.audio_buffer:
+            all_audio = self.audio_buffer.get_all_audio()
+            if len(all_audio) > 0 and not self.final_sent:
+                loop = asyncio.get_event_loop()
+                
+                if self.use_nemo:
+                    final_transcription = await loop.run_in_executor(
+                        executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+                    )
+                else:
+                    final_transcription = await loop.run_in_executor(
+                        executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+                    )
+                
+                if final_transcription.strip():
+                    await self.send_transcription(final_transcription, is_final=True)
+        
+        # Clear audio buffer
+        if self.audio_buffer:
+            self.audio_buffer.clear()
+        
+        logger.info("STT session stopped")
+    
+    async def start_new_segment(self):
+        """Start a new transcription segment"""
+        self.segment_number += 1
+        self.interim_count = 0
+        self.last_interim_time = None
+        self.accumulated_transcript = ""
+        self.final_sent = False
+        self.last_partial = ""
+        self.processing_count = 0
+        
+        if self.audio_buffer:
+            self.audio_buffer.reset_for_new_segment()
+        
+        logger.info(f"Started new transcription segment #{self.segment_number}")
+    
+    async def add_audio_data(self, audio_bytes):
+        """Add audio data to buffer"""
+        if self.audio_buffer and self.running:
+            audio_data = linear16_to_audio(audio_bytes, self.config["sample_rate"])
+            self.audio_buffer.add_audio(audio_data)
+    
+    async def _process_audio_chunks(self):
+        """Process audio chunks for interim results - with debugging"""
+        model_name = "NeMo" if self.use_nemo else "Whisper"
+        logger.info(f"Starting audio chunk processing for {model_name}")
+        
+        chunk_count = 0
+        
+        while self.running:
+            try:
+                if self.audio_buffer and self.audio_buffer.has_chunk_ready():
+                    chunk_count += 1
+                    logger.info(f"{model_name} - Processing chunk #{chunk_count}")
+                    
+                    chunk_signal = self.audio_buffer.get_chunk_for_processing()
+                    if chunk_signal is not None:
+                        all_audio = self.audio_buffer.get_all_audio()
+                        
+                        logger.info(f"{model_name} - Got {len(all_audio)} samples for processing")
+                        
+                        if len(all_audio) > 0:
+                            # Get the latest chunk for VAD check
+                            latest_chunk_start = max(0, len(all_audio) - self.audio_buffer.chunk_samples)
+                            latest_chunk = all_audio[latest_chunk_start:]
+                            
+                            # Check for speech activity
+                            has_speech = self.audio_buffer.is_speech(latest_chunk)
+                            logger.info(f"{model_name} - Speech detected: {has_speech}")
+                            
+                            if has_speech:
+                                logger.info(f"{model_name} - Starting transcription...")
+                                
+                                loop = asyncio.get_event_loop()
+                                start_time = time.time()
+                                
+                                try:
+                                    # Choose transcription method based on model selection
+                                    if self.use_nemo:
+                                        transcription = await loop.run_in_executor(
+                                            executor, transcribe_with_nemo, all_audio, self.config["sample_rate"]
+                                        )
+                                    else:
+                                        transcription = await loop.run_in_executor(
+                                            executor, transcribe_with_whisper, all_audio, self.config["sample_rate"]
+                                        )
+                                    
+                                    process_time = time.time() - start_time
+                                    logger.info(f"{model_name} - Transcription completed in {process_time:.2f}s: '{transcription}'")
+                                    
+                                    if transcription and transcription.strip():
+                                        self.processing_count += 1
+                                        self.accumulated_transcript = transcription
+                                        
+                                        if transcription != self.last_partial or self.interim_count == 0:
+                                            self.last_partial = transcription
+                                            self.interim_count += 1
+                                            self.last_interim_time = time.time()
+                                            logger.info(f"{model_name} - Updated interim_count to {self.interim_count}")
+                                        else:
+                                            self.last_interim_time = time.time()
+                                            logger.info(f"{model_name} - Same transcription, updating time only")
+                                    else:
+                                        logger.info(f"{model_name} - No transcription result")
+                                        
+                                except Exception as e:
+                                    logger.error(f"{model_name} - Transcription error: {e}")
+                                    import traceback
+                                    traceback.print_exc()
+                            else:
+                                logger.debug(f"{model_name} - No speech in chunk")
+                    else:
+                        logger.warning(f"{model_name} - Chunk signal was None")
+                else:
+                    # Log why chunk is not ready
+                    if self.audio_buffer:
+                        current_size = len(self.audio_buffer.buffer)
+                        required_size = self.audio_buffer.chunk_samples
+                        if current_size > 0:
+                            logger.debug(f"{model_name} - Buffer: {current_size}/{required_size} samples")
+                
+                await asyncio.sleep(0.1)
+                
+            except Exception as e:
+                logger.error(f"{model_name} - Error in chunk processing: {e}")
+                import traceback
+                traceback.print_exc()
+                await asyncio.sleep(1)
+
+    async def _monitor_for_auto_final(self):
+        """Monitor for auto-final conditions with model-specific timeouts"""
+        model_name = "NeMo" if self.use_nemo else "Whisper"
+        timeout = 2.0 if self.use_nemo else 3.0  # Longer timeout for Whisper
+        
+        logger.info(f"Starting auto-final monitoring for {model_name} (timeout: {timeout}s)")
+        
+        while self.running:
+            try:
+                current_time = time.time()
+                
+                if (self.interim_count >= self.min_interim_count and 
+                    self.last_interim_time is not None and 
+                    (current_time - self.last_interim_time) >= timeout and
+                    not self.final_sent and
+                    self.accumulated_transcript.strip()):
+                    
+                    silence_duration = current_time - self.last_interim_time
+                    logger.info(f"Auto-final triggered for segment #{self.segment_number} ({model_name}) - "
+                            f"Interim count: {self.interim_count}, Silence: {silence_duration:.1f}s")
+                    
+                    await self.send_transcription(self.accumulated_transcript, is_final=True)
+                    await self.start_new_segment()
+                
+                await asyncio.sleep(0.5)  # Check every 500ms
+                
+            except Exception as e:
+                logger.error(f"Error in auto-final monitoring: {e}")
+                await asyncio.sleep(0.5)
+
+
+
+    async def send_transcription(self, text, is_final=True, confidence=0.9):
+        """Send transcription in jambonz format"""
+        try:
+            # Apply number conversion only for Whisper
+            if not self.use_nemo and is_final:
+                original_text = text
+                converted_text = convert_arabic_numbers_whisper(text)
+                
+                if original_text != converted_text:
+                    logger.info(f"Whisper - Arabic numbers converted: '{original_text}' -> '{converted_text}'")
+                text = converted_text
+            
+            message = {
+                "type": "transcription",
+                "is_final": True,  # Always send as final
+                "alternatives": [
+                    {
+                        "transcript": text,
+                        "confidence": confidence
+                    }
+                ],
+                "language": self.config.get("language", "ar-EG"),
+                "channel": 1
+            }
+            
+            await self.websocket.send(json.dumps(message))
+            self.final_sent = True
+            
+            model_name = "NeMo" if self.use_nemo else "Whisper"
+            logger.info(f"Sent FINAL transcription ({model_name}): '{text}'")
+            
+        except Exception as e:
+            logger.error(f"Error sending transcription: {e}")
+    
+    async def send_error(self, error_message):
+        """Send error message in jambonz format"""
+        try:
+            message = {
+                "type": "error",
+                "error": error_message
+            }
+            await self.websocket.send(json.dumps(message))
+            logger.error(f"Sent error: {error_message}")
+        except Exception as e:
+            logger.error(f"Error sending error message: {e}")
+
+async def handle_jambonz_websocket(websocket):
+    """Handle jambonz WebSocket connections"""
+    
+    client_id = f"jambonz_{id(websocket)}"
+    logger.info(f"New unified STT connection: {client_id}")
+    
+    handler = UnifiedSTTHandler(websocket)
+    
+    try:
+        async for message in websocket:
+            try:
+                if isinstance(message, str):
+                    data = json.loads(message)
+                    message_type = data.get("type")
+                    
+                    if message_type == "start":
+                        logger.info(f"Received start message: {data}")
+                        await handler.start_processing(data)
+                        
+                    elif message_type == "stop":
+                        logger.info("Received stop message - closing WebSocket")
+                        await handler.stop_processing()
+                        await websocket.close(code=1000, reason="Session stopped by client")
+                        break
+                        
+                    else:
+                        logger.warning(f"Unknown message type: {message_type}")
+                        await handler.send_error(f"Unknown message type: {message_type}")
+                
+                else:
+                    # Handle binary audio data
+                    if not handler.running or handler.audio_buffer is None:
+                        logger.warning("Received audio data outside of active session")
+                        await handler.send_error("Received audio before start message or after stop")
+                        continue
+                    
+                    await handler.add_audio_data(message)
+            
+            except json.JSONDecodeError as e:
+                logger.error(f"JSON decode error: {e}")
+                await handler.send_error(f"Invalid JSON: {str(e)}")
+            except Exception as e:
+                logger.error(f"Error processing message: {e}")
+                await handler.send_error(f"Processing error: {str(e)}")
+    
+    except websockets.exceptions.ConnectionClosed:
+        logger.info(f"Unified STT connection closed: {client_id}")
+    except Exception as e:
+        logger.error(f"Unified STT WebSocket error: {e}")
+        try:
+            await handler.send_error(str(e))
+        except:
+            pass
+    finally:
+        if handler.running:
+            await handler.stop_processing()
+        logger.info(f"Unified STT connection ended: {client_id}")
+
+async def main():
+    """Start the Unified Arabic STT WebSocket server"""
+    logger.info("Starting Unified Arabic STT WebSocket server on port 3007...")
+    
+    # Check model availability
+    models_available = []
+    if asr_model_nemo is not None:
+        models_available.append("NeMo FastConformer (ar-EG)")
+    if whisper_model is not None:
+        models_available.append("Whisper large-v3 (ar-EG-whis)")
+    
+    if not models_available:
+        logger.error("No models available! Please check model paths and installations.")
+        return
+    
+    # Start WebSocket server
+    server = await websockets.serve(
+        handle_jambonz_websocket,
+        "0.0.0.0", 
+        3007,
+        ping_interval=20,
+        ping_timeout=10,
+        close_timeout=10
+    )
+    
+    logger.info("Unified Arabic STT WebSocket server started on ws://0.0.0.0:3007")
+    logger.info("Ready to handle jambonz STT requests with both models")
+    logger.info("ROUTING:")
+    logger.info("- language: 'ar-EG' → NeMo FastConformer (with built-in number conversion)")
+    logger.info("- language: 'ar-EG-whis' → Whisper large-v3 (with pyarabic number conversion)")
+    logger.info("FEATURES:")
+    logger.info("- Continuous transcription with segmentation")
+    logger.info("- Voice Activity Detection")
+    logger.info("- Auto-final detection (2s silence timeout)")
+    logger.info("- Model-specific number conversion")
+    logger.info(f"AVAILABLE MODELS: {', '.join(models_available)}")
+    
+    # Wait for the server to close
+    await server.wait_closed()
+
+if __name__ == "__main__":
+    print("=" * 80)
+    print("Unified Arabic STT Server (NeMo + Whisper)")
+    print("=" * 80)
+    print("WebSocket Port: 3007")
+    print("Protocol: jambonz STT API")
+    print("Audio Format: LINEAR16 PCM @ 8kHz → 16kHz")
+    print()
+    print("LANGUAGE ROUTING:")
+    print("- 'ar-EG' → NeMo FastConformer")
+    print("  • Built-in Arabic number word to digit conversion")
+    print("  • Optimized for Arabic dialects")
+    print("- 'ar-EG-whis' → Whisper large-v3")
+    print("  • pyarabic library number conversion (final transcripts only)")
+    print("  • OpenAI Whisper model")
+    print()
+    print("FEATURES:")
+    print("- Automatic model selection based on language parameter")
+    print("- Voice Activity Detection")
+    print("- Auto-final detection (2 seconds silence)")
+    print("- Model-specific number conversion strategies")
+    print("- Continuous transcription with segmentation")
+    print()
+    
+    # Check model availability for startup info
+    nemo_status = "✓ Available" if asr_model_nemo is not None else "✗ Not Available"
+    whisper_status = "✓ Available" if whisper_model is not None else "✗ Not Available"
+    arabic_numbers_status = "✓ Available" if arabic_numbers_available else "✗ Not Available (install pyarabic)"
+    
+    print("MODEL STATUS:")
+    print(f"- NeMo FastConformer: {nemo_status}")
+    print(f"- Whisper large-v3: {whisper_status}")
+    print(f"- pyarabic (Whisper numbers): {arabic_numbers_status}")
+    print("=" * 80)
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        print("\nShutting down unified server...")
+    except Exception as e:
+        print(f"Server error: {e}")

whisper_checkpoints/models--openai--whisper-large-v2/blobs/1ce74630ed587e80f3db2b3d434f7026327f131e

@@ -0,0 +1,144 @@
+{
+  "_name_or_path": "openai/whisper-large-v2",
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "architectures": [
+    "WhisperForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "begin_suppress_tokens": [
+    220,
+    50257
+  ],
+  "bos_token_id": 50257,
+  "d_model": 1280,
+  "decoder_attention_heads": 20,
+  "decoder_ffn_dim": 5120,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 32,
+  "decoder_start_token_id": 50258,
+  "dropout": 0.0,
+  "encoder_attention_heads": 20,
+  "encoder_ffn_dim": 5120,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 32,
+  "eos_token_id": 50257,
+  "forced_decoder_ids": [
+    [
+      1,
+      50259
+    ],
+    [
+      2,
+      50359
+    ],
+    [
+      3,
+      50363
+    ]
+  ],
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "max_length": 448,
+  "max_source_positions": 1500,
+  "max_target_positions": 448,
+  "model_type": "whisper",
+  "num_hidden_layers": 32,
+  "num_mel_bins": 80,
+  "pad_token_id": 50257,
+  "scale_embedding": false,
+  "suppress_tokens": [
+    1,
+    2,
+    7,
+    8,
+    9,
+    10,
+    14,
+    25,
+    26,
+    27,
+    28,
+    29,
+    31,
+    58,
+    59,
+    60,
+    61,
+    62,
+    63,
+    90,
+    91,
+    92,
+    93,
+    359,
+    503,
+    522,
+    542,
+    873,
+    893,
+    902,
+    918,
+    922,
+    931,
+    1350,
+    1853,
+    1982,
+    2460,
+    2627,
+    3246,
+    3253,
+    3268,
+    3536,
+    3846,
+    3961,
+    4183,
+    4667,
+    6585,
+    6647,
+    7273,
+    9061,
+    9383,
+    10428,
+    10929,
+    11938,
+    12033,
+    12331,
+    12562,
+    13793,
+    14157,
+    14635,
+    15265,
+    15618,
+    16553,
+    16604,
+    18362,
+    18956,
+    20075,
+    21675,
+    22520,
+    26130,
+    26161,
+    26435,
+    28279,
+    29464,
+    31650,
+    32302,
+    32470,
+    36865,
+    42863,
+    47425,
+    49870,
+    50254,
+    50258,
+    50358,
+    50359,
+    50360,
+    50361,
+    50362
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.0.dev0",
+  "use_cache": true,
+  "vocab_size": 51865
+}

whisper_checkpoints/models--openai--whisper-large-v2/blobs/57a1ba2a82c093cabff2541409ae778c97145378b9ddfa722763cb1cb8f9020b

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57a1ba2a82c093cabff2541409ae778c97145378b9ddfa722763cb1cb8f9020b
+size 6173370152

whisper_checkpoints/models--openai--whisper-large-v2/refs/main

@@ -0,0 +1 @@
+ae4642769ce2ad8fc292556ccea8e901f1530655

whisper_checkpoints/models--openai--whisper-large-v2/snapshots/ae4642769ce2ad8fc292556ccea8e901f1530655/config.json

@@ -0,0 +1,144 @@
+{
+  "_name_or_path": "openai/whisper-large-v2",
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "architectures": [
+    "WhisperForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "begin_suppress_tokens": [
+    220,
+    50257
+  ],
+  "bos_token_id": 50257,
+  "d_model": 1280,
+  "decoder_attention_heads": 20,
+  "decoder_ffn_dim": 5120,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 32,
+  "decoder_start_token_id": 50258,
+  "dropout": 0.0,
+  "encoder_attention_heads": 20,
+  "encoder_ffn_dim": 5120,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 32,
+  "eos_token_id": 50257,
+  "forced_decoder_ids": [
+    [
+      1,
+      50259
+    ],
+    [
+      2,
+      50359
+    ],
+    [
+      3,
+      50363
+    ]
+  ],
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "max_length": 448,
+  "max_source_positions": 1500,
+  "max_target_positions": 448,
+  "model_type": "whisper",
+  "num_hidden_layers": 32,
+  "num_mel_bins": 80,
+  "pad_token_id": 50257,
+  "scale_embedding": false,
+  "suppress_tokens": [
+    1,
+    2,
+    7,
+    8,
+    9,
+    10,
+    14,
+    25,
+    26,
+    27,
+    28,
+    29,
+    31,
+    58,
+    59,
+    60,
+    61,
+    62,
+    63,
+    90,
+    91,
+    92,
+    93,
+    359,
+    503,
+    522,
+    542,
+    873,
+    893,
+    902,
+    918,
+    922,
+    931,
+    1350,
+    1853,
+    1982,
+    2460,
+    2627,
+    3246,
+    3253,
+    3268,
+    3536,
+    3846,
+    3961,
+    4183,
+    4667,
+    6585,
+    6647,
+    7273,
+    9061,
+    9383,
+    10428,
+    10929,
+    11938,
+    12033,
+    12331,
+    12562,
+    13793,
+    14157,
+    14635,
+    15265,
+    15618,
+    16553,
+    16604,
+    18362,
+    18956,
+    20075,
+    21675,
+    22520,
+    26130,
+    26161,
+    26435,
+    28279,
+    29464,
+    31650,
+    32302,
+    32470,
+    36865,
+    42863,
+    47425,
+    49870,
+    50254,
+    50258,
+    50358,
+    50359,
+    50360,
+    50361,
+    50362
+  ],
+  "torch_dtype": "float32",
+  "transformers_version": "4.27.0.dev0",
+  "use_cache": true,
+  "vocab_size": 51865
+}

whisper_checkpoints/models--openai--whisper-large-v2/snapshots/ae4642769ce2ad8fc292556ccea8e901f1530655/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57a1ba2a82c093cabff2541409ae778c97145378b9ddfa722763cb1cb8f9020b
+size 6173370152