Update app.py

app.py

@@ -133,7 +133,7 @@ class DatabaseManager:
         with sqlite3.connect(self.db_path) as conn:
             cursor = conn.cursor()
             
-            # Cache table
+            # Cache table - now includes english_reference
             cursor.execute('''
                 CREATE TABLE IF NOT EXISTS cache (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -141,12 +141,20 @@ class DatabaseManager:
                     filename TEXT,
                     file_size INTEGER,
                     transcription TEXT,
+                    english_reference TEXT,
                     language TEXT,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                     last_accessed TIMESTAMP DEFAULT CURRENT_TIMESTAMP
                 )
             ''')
             
+            # Add english_reference column if it doesn't exist (for existing databases)
+            cursor.execute("PRAGMA table_info(cache)")
+            columns = [column[1] for column in cursor.fetchall()]
+            if 'english_reference' not in columns:
+                cursor.execute('ALTER TABLE cache ADD COLUMN english_reference TEXT')
+                logger.info("Added english_reference column to cache table")
+            
             # Processing status table
             cursor.execute('''
                 CREATE TABLE IF NOT EXISTS processing_status (
@@ -162,24 +170,32 @@ class DatabaseManager:
                 )
             ''')
             
-            # Translation cache table
+            # Translation cache table - now includes source_language
             cursor.execute('''
                 CREATE TABLE IF NOT EXISTS translation_cache (
                     id INTEGER PRIMARY KEY AUTOINCREMENT,
                     text_hash TEXT,
+                    source_language TEXT DEFAULT 'english',
                     target_language TEXT,
                     translated_text TEXT,
                     created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
                     last_accessed TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-                    UNIQUE(text_hash, target_language)
+                    UNIQUE(text_hash, source_language, target_language)
                 )
             ''')
             
+            # Add source_language column if it doesn't exist (for existing databases)
+            cursor.execute("PRAGMA table_info(translation_cache)")
+            columns = [column[1] for column in cursor.fetchall()]
+            if 'source_language' not in columns:
+                cursor.execute('ALTER TABLE translation_cache ADD COLUMN source_language TEXT DEFAULT "english"')
+                logger.info("Added source_language column to translation_cache table")
+            
             # Create indexes for better performance
             cursor.execute('CREATE INDEX IF NOT EXISTS idx_cache_hash ON cache(file_hash)')
             cursor.execute('CREATE INDEX IF NOT EXISTS idx_cache_created ON cache(created_at)')
             cursor.execute('CREATE INDEX IF NOT EXISTS idx_status_hash ON processing_status(file_hash)')
-            cursor.execute('CREATE INDEX IF NOT EXISTS idx_translation_hash ON translation_cache(text_hash, target_language)')
+            cursor.execute('CREATE INDEX IF NOT EXISTS idx_translation_hash ON translation_cache(text_hash, source_language, target_language)')
             
             conn.commit()
     
@@ -203,13 +219,13 @@ def calculate_text_hash(text: str) -> str:
     """Calculate hash for text"""
     return hashlib.md5(text.encode('utf-8')).hexdigest()
 
-async def get_from_cache(file_hash: str) -> Optional[str]:
-    """Get transcription from cache"""
+async def get_from_cache(file_hash: str) -> Optional[Dict[str, Any]]:
+    """Get transcription from cache - now returns both original and english reference"""
     try:
         with db_manager.get_connection() as conn:
             cursor = conn.cursor()
             cursor.execute(
-                'SELECT transcription FROM cache WHERE file_hash = ?',
+                'SELECT transcription, english_reference FROM cache WHERE file_hash = ?',
                 (file_hash,)
             )
             result = cursor.fetchone()
@@ -220,29 +236,63 @@ async def get_from_cache(file_hash: str) -> Optional[str]:
                     (file_hash,)
                 )
                 conn.commit()
-                return result[0]
+                
+                # Parse the cached transcription data
+                try:
+                    transcription_data = json.loads(result[0]) if result[0] else {}
+                except:
+                    transcription_data = {"text": result[0] or ""}
+                
+                # Add english reference if available
+                if result[1]:
+                    transcription_data["english_reference"] = result[1]
+                
+                return transcription_data
         return None
     except Exception as e:
         logger.error(f"Error getting from cache: {e}")
         return None
 
-async def get_translation_from_cache(text_hash: str, target_language: str) -> Optional[str]:
+async def get_english_reference_from_cache(file_hash: str) -> Optional[str]:
+    """Get English reference text for a cached audio file"""
+    try:
+        with db_manager.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                'SELECT english_reference FROM cache WHERE file_hash = ?',
+                (file_hash,)
+            )
+            result = cursor.fetchone()
+            
+            if result and result[0]:
+                cursor.execute(
+                    'UPDATE cache SET last_accessed = CURRENT_TIMESTAMP WHERE file_hash = ?',
+                    (file_hash,)
+                )
+                conn.commit()
+                return result[0]
+        return None
+    except Exception as e:
+        logger.error(f"Error getting english reference from cache: {e}")
+        return None
+
+async def get_translation_from_cache(text_hash: str, target_language: str, source_language: str = "english") -> Optional[str]:
     """Get translation from cache"""
     try:
         with db_manager.get_connection() as conn:
             cursor = conn.cursor()
             cursor.execute(
                 '''SELECT translated_text FROM translation_cache 
-                   WHERE text_hash = ? AND target_language = ?''',
-                (text_hash, target_language)
+                   WHERE text_hash = ? AND target_language = ? AND source_language = ?''',
+                (text_hash, target_language, source_language)
             )
             result = cursor.fetchone()
             
             if result:
                 cursor.execute(
                     '''UPDATE translation_cache SET last_accessed = CURRENT_TIMESTAMP 
-                       WHERE text_hash = ? AND target_language = ?''',
-                    (text_hash, target_language)
+                       WHERE text_hash = ? AND target_language = ? AND source_language = ?''',
+                    (text_hash, target_language, source_language)
                 )
                 conn.commit()
                 return result[0]
@@ -251,31 +301,31 @@ async def get_translation_from_cache(text_hash: str, target_language: str) -> Op
         logger.error(f"Error getting translation from cache: {e}")
         return None
 
-async def save_to_cache(file_hash: str, filename: str, file_size: int, transcription: str, language: str = None):
-    """Save transcription to cache"""
+async def save_to_cache(file_hash: str, filename: str, file_size: int, transcription: str, english_reference: str = None, language: str = None):
+    """Save transcription to cache - now includes english reference"""
     try:
         with db_manager.get_connection() as conn:
             cursor = conn.cursor()
             cursor.execute(
                 '''INSERT OR REPLACE INTO cache 
-                   (file_hash, filename, file_size, transcription, language) 
-                   VALUES (?, ?, ?, ?, ?)''',
-                (file_hash, filename, file_size, transcription, language)
+                   (file_hash, filename, file_size, transcription, english_reference, language) 
+                   VALUES (?, ?, ?, ?, ?, ?)''',
+                (file_hash, filename, file_size, transcription, english_reference, language)
             )
             conn.commit()
     except Exception as e:
         logger.error(f"Error saving to cache: {e}")
 
-async def save_translation_to_cache(text_hash: str, target_language: str, translated_text: str):
+async def save_translation_to_cache(text_hash: str, target_language: str, translated_text: str, source_language: str = "english"):
     """Save translation to cache"""
     try:
         with db_manager.get_connection() as conn:
             cursor = conn.cursor()
             cursor.execute(
                 '''INSERT OR REPLACE INTO translation_cache 
-                   (text_hash, target_language, translated_text) 
-                   VALUES (?, ?, ?)''',
-                (text_hash, target_language, translated_text)
+                   (text_hash, source_language, target_language, translated_text) 
+                   VALUES (?, ?, ?, ?)''',
+                (text_hash, source_language, target_language, translated_text)
             )
             conn.commit()
     except Exception as e:
@@ -368,13 +418,13 @@ def estimate_processing_time(file_size_mb: float) -> int:
     return max(1, int(estimated_seconds / 60))
 
 async def background_transcription(file_path: str, file_hash: str, filename: str, file_size: int, translate_to_english: bool = False):
-    """Background task for transcription"""
+    """Background task for transcription with English reference caching"""
     try:
         logger.info(f"Starting background transcription for {filename}")
         
         await update_processing_status(file_hash, status='processing', progress=10)
         
-        # Transcribe audio
+        # Transcribe audio in original language
         result = whisper_model.transcribe(
             file_path, 
             fp16=(device != "cpu"),
@@ -384,7 +434,7 @@ async def background_transcription(file_path: str, file_hash: str, filename: str
             word_timestamps=False
         )
         
-        await update_processing_status(file_hash, progress=60)
+        await update_processing_status(file_hash, progress=40)
         
         text = result["text"].strip() or "No text detected"
         detected_language = result.get("language", "unknown")
@@ -395,27 +445,39 @@ async def background_transcription(file_path: str, file_hash: str, filename: str
             "from_cache": False
         }
         
-        # Translate if requested and needed
-        if translate_to_english and detected_language != "en":
-            await update_processing_status(file_hash, progress=80)
+        # Always get English reference if not already English
+        english_reference = None
+        if detected_language != "en":
+            await update_processing_status(file_hash, progress=70)
             
+            # Get English translation using Whisper's translate task
             english_result = whisper_model.transcribe(
                 file_path, 
                 fp16=(device != "cpu"),
                 language=None,
-                task="translate",
+                task="translate",  # This translates to English
                 verbose=False,
                 word_timestamps=False
             )
             
-            english_text = english_result["text"].strip()
-            if english_text:
-                response_data["english_text"] = english_text
+            english_reference = english_result["text"].strip()
+            if english_reference:
+                response_data["english_reference"] = english_reference
+                
+                # If user requested English translation, include it
+                if translate_to_english:
+                    response_data["english_text"] = english_reference
+        else:
+            # If the original is English, use it as reference
+            english_reference = text
+            response_data["english_reference"] = english_reference
         
-        # Save to cache
+        await update_processing_status(file_hash, progress=90)
+        
+        # Save to cache with English reference
         await save_to_cache(
             file_hash, filename, file_size, 
-            json.dumps(response_data), detected_language
+            json.dumps(response_data), english_reference, detected_language
         )
         
         await update_processing_status(file_hash, status='completed', progress=100)
@@ -433,236 +495,41 @@ async def background_transcription(file_path: str, file_hash: str, filename: str
         except Exception as e:
             logger.error(f"Error deleting temp file: {e}")
 
-def split_text_smartly(text: str, max_tokens: int = 400) -> list:
-    """Improved text splitting that handles various languages and formats"""
-    # First try to split by sentences (multiple patterns for different languages)
-    sentence_patterns = [
-        r'(?<=[.!?])\s+',  # English
-        r'(?<=[。！？])\s*',  # Chinese/Japanese
-        r'(?<=[۔؟!])\s+',   # Persian/Arabic
-        r'(?<=[\.!?])\s+'   # Fallback
-    ]
-    
-    sentences = []
-    remaining_text = text
-    
-    for pattern in sentence_patterns:
-        try:
-            potential_sentences = re.split(pattern, remaining_text)
-            if len(potential_sentences) > 1:
-                sentences = potential_sentences
-                break
-        except:
-            continue
-    
-    # If no sentence splitting worked, split by length
-    if not sentences or len(sentences) == 1:
-        chunk_size = 200  # Conservative chunk size
-        sentences = [text[i:i + chunk_size] for i in range(0, len(text), chunk_size)]
-    
-    # Group sentences into chunks that fit token limit
-    chunks = []
-    current_chunk = []
-    current_length = 0
-    
-    for sentence in sentences:
-        if not sentence.strip():
-            continue
-            
-        try:
-            sentence_tokens = len(translation_tokenizer.tokenize(sentence))
-        except:
-            # Estimate tokens if tokenizer fails
-            sentence_tokens = len(sentence.split()) * 1.3
-            
-        if current_length + sentence_tokens > max_tokens and current_chunk:
-            chunks.append(' '.join(current_chunk).strip())
-            current_chunk = [sentence]
-            current_length = sentence_tokens
-        else:
-            current_chunk.append(sentence)
-            current_length += sentence_tokens
-    
-    if current_chunk:
-        chunks.append(' '.join(current_chunk).strip())
-    
-    # Remove empty chunks
-    chunks = [chunk for chunk in chunks if chunk.strip()]
-    
-    return chunks if chunks else [text]
-
-def translate_text_chunk(text: str, target_code: str, max_retries: int = 3) -> str:
-    """Improved translation with retry logic and better error handling"""
-    if not text.strip():
-        return ""
-        
-    for attempt in range(max_retries):
-        try:
-            # Use longer max_length for better translation quality
-            max_length = min(1024, len(text) * 2)  # Dynamic max length
-            
-            inputs = translation_tokenizer(
-                text, 
-                return_tensors="pt", 
-                padding=True, 
-                truncation=True, 
-                max_length=512  # Input limit
-            )
-            
-            if device == "cuda":
-                inputs = {k: v.to(device) for k, v in inputs.items()}
-            
-            # Generate translation with better parameters
-            translated_tokens = translation_model.generate(
-                **inputs, 
-                forced_bos_token_id=translation_tokenizer.lang_code_to_id[target_code],
-                max_length=max_length,  # Use dynamic max length
-                min_length=5,  # Ensure minimum output
-                num_beams=4,
-                early_stopping=True,
-                do_sample=False,  # Deterministic output
-                temperature=1.0,
-                repetition_penalty=1.1
-            )
-            
-            translated_text = translation_tokenizer.batch_decode(
-                translated_tokens, 
-                skip_special_tokens=True
-            )[0].strip()
-            
-            if translated_text and len(translated_text) > 2:
-                return translated_text
-            else:
-                logger.warning(f"Empty translation on attempt {attempt + 1}")
-                
-        except Exception as e:
-            logger.error(f"Translation attempt {attempt + 1} failed: {e}")
-            if attempt == max_retries - 1:
-                return f"[Translation failed: {text[:50]}...]"
-    
-    return f"[Translation failed after {max_retries} attempts]"
-
-async def debug_translate_endpoint(
-    text: str = Form(..., min_length=1),
-    target_language: str = Form(...)
-):
-    """Debug version of translation endpoint with detailed logging"""
-    
-    if not translation_model:
-        raise HTTPException(status_code=503, detail="Translation service not available")
-    
-    text = text.strip()
-    logger.info(f"=== TRANSLATION DEBUG START ===")
-    logger.info(f"Original text length: {len(text)} characters")
-    logger.info(f"Original text preview: {text[:200]}...")
-    logger.info(f"Target language: {target_language}")
-    
-    target_language_lower = target_language.lower()
-    if target_language_lower not in LANGUAGE_CODES:
-        raise HTTPException(
-            status_code=400, 
-            detail=f"Unsupported language. Supported: {list(LANGUAGE_CODES.keys())}"
-        )
-    
-    # Check cache
-    text_hash = calculate_text_hash(text)
-    cached_translation = await get_translation_from_cache(text_hash, target_language_lower)
-    if cached_translation:
-        logger.info("Returning cached translation")
-        return JSONResponse({
-            "text": text,
-            "translated_text": cached_translation,
-            "target_language": target_language,
-            "from_cache": True
-        })
-    
+def translate_text_chunk(text: str, target_code: str) -> str:
+    """Translate a single chunk of text"""
     try:
-        target_code = LANGUAGE_CODES[target_language_lower]
-        
-        # Smart text splitting with debug info
-        chunks = split_text_smartly(text, max_tokens=350)
-        logger.info(f"Text split into {len(chunks)} chunks")
-        
-        for i, chunk in enumerate(chunks):
-            logger.info(f"Chunk {i+1}: length={len(chunk)}, preview='{chunk[:100]}...'")
-        
-        translated_chunks = []
-        debug_info = {
-            "total_chunks": len(chunks),
-            "successful_chunks": 0,
-            "failed_chunks": 0,
-            "chunk_details": []
-        }
-        
-        for i, chunk in enumerate(chunks):
-            if not chunk.strip():
-                continue
-            
-            chunk_info = {
-                "chunk_id": i+1,
-                "original_length": len(chunk),
-                "original_preview": chunk[:100]
-            }
-            
-            logger.info(f"Processing chunk {i+1}/{len(chunks)}")
-            
-            translated_chunk = translate_text_chunk(chunk, target_code)
-            
-            chunk_info["translated_preview"] = translated_chunk[:100] if translated_chunk else "FAILED"
-            chunk_info["translated_length"] = len(translated_chunk) if translated_chunk else 0
-            
-            if translated_chunk and not translated_chunk.startswith("[Translation"):
-                translated_chunks.append(translated_chunk)
-                debug_info["successful_chunks"] += 1
-                chunk_info["status"] = "success"
-                logger.info(f"Chunk {i+1} translated successfully: {len(translated_chunk)} chars")
-            else:
-                debug_info["failed_chunks"] += 1
-                chunk_info["status"] = "failed"
-                logger.error(f"Chunk {i+1} translation failed: {translated_chunk}")
-            
-            debug_info["chunk_details"].append(chunk_info)
-        
-        if not translated_chunks:
-            logger.error("All translation chunks failed!")
-            raise HTTPException(status_code=500, detail="Translation failed for all text chunks")
-        
-        # Combine translated chunks
-        translated_text = ' '.join(translated_chunks)
-        logger.info(f"Combined translation length: {len(translated_text)} characters")
-        logger.info(f"Translation preview: {translated_text[:200]}...")
-        
-        # Clean up the translation
-        original_length = len(translated_text)
-        translated_text = re.sub(r'\s+', ' ', translated_text).strip()
-        logger.info(f"After cleanup: {len(translated_text)} characters (was {original_length})")
-        
-        # Quality check
-        translation_ratio = len(translated_text) / len(text) if len(text) > 0 else 0
-        logger.info(f"Translation ratio: {translation_ratio:.2f} (translated/original)")
+        # Tokenize input
+        inputs = translation_tokenizer(
+            text, 
+            return_tensors="pt", 
+            padding=True, 
+            truncation=True, 
+            max_length=512
+        )
         
-        if translation_ratio < 0.1:
-            logger.warning(f"Translation seems too short! Ratio: {translation_ratio}")
+        if device == "cuda":
+            inputs = {k: v.to(device) for k, v in inputs.items()}
         
-        # Save to cache
-        await save_translation_to_cache(text_hash, target_language_lower, translated_text)
+        # Generate translation
+        translated_tokens = translation_model.generate(
+            **inputs, 
+            forced_bos_token_id=translation_tokenizer.lang_code_to_id[target_code],
+            max_length=512,
+            num_beams=5,
+            early_stopping=True
+        )
         
-        logger.info("=== TRANSLATION DEBUG END ===")
+        # Decode output
+        translated_text = translation_tokenizer.batch_decode(
+            translated_tokens, 
+            skip_special_tokens=True
+        )[0].strip()
         
-        return JSONResponse({
-            "text": text,
-            "translated_text": translated_text,
-            "target_language": target_language,
-            "from_cache": False,
-            "debug_info": debug_info,
-            "translation_ratio": translation_ratio
-        })
+        return translated_text
         
-    except HTTPException:
-        raise
     except Exception as e:
-        logger.error(f"Translation error: {e}")
-        raise HTTPException(status_code=500, detail=f"Translation failed: {str(e)}")
+        logger.error(f"Error translating chunk: {e}")
+        return f"[Translation error: {str(e)}]"
 
 @app.get("/")
 async def root():
@@ -679,12 +546,17 @@ async def root():
             
             cursor.execute('SELECT COUNT(*) FROM translation_cache')
             translation_cache_count = cursor.fetchone()[0] or 0
+            
+            # Count cached English references
+            cursor.execute('SELECT COUNT(*) FROM cache WHERE english_reference IS NOT NULL')
+            english_reference_count = cursor.fetchone()[0] or 0
         
         return {
             "message": "Whisper API with Translation is running", 
             "device": device,
             "cuda_available": torch.cuda.is_available(),
             "cached_files": cache_count,
+            "english_references": english_reference_count,
             "translation_cache": translation_cache_count,
             "currently_processing": processing_count,
             "translation_available": translation_model is not None,
@@ -701,7 +573,7 @@ async def transcribe_audio(
     translate_to_english: bool = False,
     language: Optional[str] = Query(None, description="Specify language code for better accuracy")
 ):
-    """Transcribe audio file to text"""
+    """Transcribe audio file to text with English reference caching"""
     tmp_file_path = None
     
     try:
@@ -733,7 +605,7 @@ async def transcribe_audio(
         if cached_result:
             logger.info("Cache hit - returning cached result")
             await remove_processing_status(file_hash)
-            return JSONResponse(json.loads(cached_result))
+            return JSONResponse(cached_result)
         
         # Check if already processing
         processing_status = await get_processing_status(file_hash)
@@ -783,18 +655,28 @@ async def transcribe_audio(
                     "from_cache": False
                 }
                 
-                # Translate if requested
-                if translate_to_english and detected_language != "en":
+                # Always get English reference
+                english_reference = None
+                if detected_language != "en":
+                    # Get English translation using Whisper's translate task
                     transcribe_args['task'] = "translate"
                     english_result = whisper_model.transcribe(tmp_file_path, **transcribe_args)
-                    english_text = english_result["text"].strip()
-                    if english_text:
-                        response_data["english_text"] = english_text
+                    english_reference = english_result["text"].strip()
+                    if english_reference:
+                        response_data["english_reference"] = english_reference
+                        
+                        # If user requested English translation, include it
+                        if translate_to_english:
+                            response_data["english_text"] = english_reference
+                else:
+                    # If original is English, use it as reference
+                    english_reference = text
+                    response_data["english_reference"] = english_reference
                 
-                # Save to cache
+                # Save to cache with English reference
                 await save_to_cache(
                     file_hash, file.filename, file_size, 
-                    json.dumps(response_data), detected_language
+                    json.dumps(response_data), english_reference, detected_language
                 )
                 
                 return JSONResponse(response_data)
@@ -836,9 +718,10 @@ async def transcribe_audio(
 @app.post("/translate")
 async def translate_endpoint(
     text: str = Form(..., min_length=1),
-    target_language: str = Form(...)
+    target_language: str = Form(...),
+    file_hash: Optional[str] = Form(None, description="Hash of audio file for using English reference")
 ):
-    """Improved translation endpoint with better chunking and error handling"""
+    """Translate text to target language - preferably using English reference from audio"""
     
     if not translation_model:
         raise HTTPException(status_code=503, detail="Translation service not available")
@@ -854,76 +737,272 @@ async def translate_endpoint(
             detail=f"Unsupported language. Supported: {list(LANGUAGE_CODES.keys())}"
         )
     
-    # Check cache
-    text_hash = calculate_text_hash(text)
-    cached_translation = await get_translation_from_cache(text_hash, target_language_lower)
+    # If target language is English, check if we have English reference from audio
+    if target_language_lower == 'english' and file_hash:
+        english_reference = await get_english_reference_from_cache(file_hash)
+        if english_reference:
+            return JSONResponse({
+                "text": text,
+                "translated_text": english_reference,
+                "target_language": target_language,
+                "from_cache": True,
+                "source": "whisper_english_reference"
+            })
+    
+    # Determine source text for translation
+    source_text = text
+    source_language = "unknown"
+    
+    # If we have a file_hash, try to use English reference for better translation
+    if file_hash:
+        english_reference = await get_english_reference_from_cache(file_hash)
+        if english_reference and target_language_lower != 'english':
+            source_text = english_reference
+            source_language = "english"
+            logger.info(f"Using English reference for translation to {target_language}")
+    
+    # Check translation cache
+    text_hash = calculate_text_hash(source_text)
+    cached_translation = await get_translation_from_cache(text_hash, target_language_lower, source_language)
     if cached_translation:
         return JSONResponse({
             "text": text,
             "translated_text": cached_translation,
             "target_language": target_language,
-            "from_cache": True
+            "from_cache": True,
+            "source_language": source_language
         })
     
+    # Perform translation
     try:
         target_code = LANGUAGE_CODES[target_language_lower]
         
-        # Smart text splitting
-        chunks = split_text_smartly(text, max_tokens=350)  # Conservative token limit
-        logger.info(f"Split text into {len(chunks)} chunks for translation")
+        # Split text into sentences for better translation
+        sentences = re.split(r'(?<=[.!?])\s+', source_text)
+        logger.info(f"Split text into {len(sentences)} sentences for translation")
         
-        translated_chunks = []
+        translated_sentences = []
         
-        for i, chunk in enumerate(chunks):
-            if not chunk.strip():
+        for i, sentence in enumerate(sentences):
+            if not sentence.strip():
                 continue
                 
-            logger.info(f"Translating chunk {i+1}/{len(chunks)}: '{chunk[:100]}...'")
-            
-            translated_chunk = translate_text_chunk(chunk, target_code)
+            logger.info(f"Translating sentence {i+1}/{len(sentences)}: '{sentence[:50]}...'")
             
-            if translated_chunk and not translated_chunk.startswith("[Translation"):
-                translated_chunks.append(translated_chunk)
-            else:
-                logger.error(f"Failed to translate chunk {i+1}: {chunk[:50]}...")
-                # Try to translate smaller pieces of the failed chunk
-                smaller_chunks = split_text_smartly(chunk, max_tokens=200)
-                for small_chunk in smaller_chunks:
-                    small_translation = translate_text_chunk(small_chunk, target_code)
-                    if small_translation and not small_translation.startswith("[Translation"):
-                        translated_chunks.append(small_translation)
+            try:
+                # Tokenize input
+                inputs = translation_tokenizer(
+                    sentence, 
+                    return_tensors="pt", 
+                    padding=True, 
+                    truncation=True, 
+                    max_length=512
+                )
+                
+                if device == "cuda":
+                    inputs = {k: v.to(device) for k, v in inputs.items()}
+                
+                # Generate translation
+                translated_tokens = translation_model.generate(
+                    **inputs, 
+                    forced_bos_token_id=translation_tokenizer.lang_code_to_id[target_code],
+                    max_length=512,
+                    num_beams=4,
+                    early_stopping=True
+                )
+                
+                # Decode output
+                translated_sentence = translation_tokenizer.batch_decode(
+                    translated_tokens, 
+                    skip_special_tokens=True
+                )[0].strip()
+                
+                if translated_sentence:
+                    translated_sentences.append(translated_sentence)
+                else:
+                    translated_sentences.append(f"[Translation failed for: {sentence}]")
+                    
+            except Exception as e:
+                logger.error(f"Error translating sentence {i+1}: {e}")
+                translated_sentences.append(f"[Translation error: {str(e)}]")
         
-        if not translated_chunks:
-            raise HTTPException(status_code=500, detail="Translation failed for all text chunks")
+        # Combine translated sentences
+        translated_text = ' '.join(translated_sentences)
         
-        # Combine translated chunks with proper spacing
-        translated_text = ' '.join(translated_chunks)
+        if not translated_text or translated_text.startswith("[Translation error"):
+            raise HTTPException(status_code=500, detail="Translation returned empty result or encountered errors")
         
-        # Clean up the translation
+        # Clean up translation
         translated_text = re.sub(r'\s+', ' ', translated_text).strip()
-        translated_text = re.sub(r'([.!?])\s*([.!?])', r'\1 \2', translated_text)  # Fix punctuation
-        
-        if len(translated_text) < len(text) * 0.1:  # Sanity check
-            logger.warning("Translation seems too short compared to original")
         
         # Save to cache
-        await save_translation_to_cache(text_hash, target_language_lower, translated_text)
+        await save_translation_to_cache(text_hash, target_language_lower, translated_text, source_language)
         
         return JSONResponse({
             "text": text,
             "translated_text": translated_text,
             "target_language": target_language,
             "from_cache": False,
-            "chunks_processed": len(chunks),
-            "chunks_translated": len(translated_chunks)
+            "source_language": source_language,
+            "used_english_reference": bool(file_hash and source_language == "english")
         })
         
-    except HTTPException:
-        raise
     except Exception as e:
         logger.error(f"Translation error: {e}")
         raise HTTPException(status_code=500, detail=f"Translation failed: {str(e)}")
 
+@app.post("/translate_from_audio")
+async def translate_from_audio_endpoint(
+    file_hash: str = Form(...),
+    target_language: str = Form(...)
+):
+    """Translate audio content using cached English reference for better results"""
+    
+    if not translation_model:
+        raise HTTPException(status_code=503, detail="Translation service not available")
+    
+    target_language_lower = target_language.lower()
+    if target_language_lower not in LANGUAGE_CODES:
+        raise HTTPException(
+            status_code=400, 
+            detail=f"Unsupported language. Supported: {list(LANGUAGE_CODES.keys())}"
+        )
+    
+    # Get English reference from cache
+    english_reference = await get_english_reference_from_cache(file_hash)
+    if not english_reference:
+        raise HTTPException(status_code=404, detail="Audio file not found in cache or no English reference available")
+    
+    # If target is English, return the English reference directly
+    if target_language_lower == 'english':
+        return JSONResponse({
+            "file_hash": file_hash,
+            "translated_text": english_reference,
+            "target_language": target_language,
+            "from_cache": True,
+            "source": "whisper_english_reference"
+        })
+    
+    # Check translation cache
+    text_hash = calculate_text_hash(english_reference)
+    cached_translation = await get_translation_from_cache(text_hash, target_language_lower, "english")
+    if cached_translation:
+        return JSONResponse({
+            "file_hash": file_hash,
+            "translated_text": cached_translation,
+            "target_language": target_language,
+            "from_cache": True,
+            "source_language": "english"
+        })
+    
+    # Perform translation from English reference
+    try:
+        target_code = LANGUAGE_CODES[target_language_lower]
+        
+        # Split text into sentences for better translation
+        sentences = re.split(r'(?<=[.!?])\s+', english_reference)
+        logger.info(f"Translating from English reference - {len(sentences)} sentences to {target_language}")
+        
+        translated_sentences = []
+        
+        for i, sentence in enumerate(sentences):
+            if not sentence.strip():
+                continue
+                
+            try:
+                # Tokenize input
+                inputs = translation_tokenizer(
+                    sentence, 
+                    return_tensors="pt", 
+                    padding=True, 
+                    truncation=True, 
+                    max_length=512
+                )
+                
+                if device == "cuda":
+                    inputs = {k: v.to(device) for k, v in inputs.items()}
+                
+                # Generate translation
+                translated_tokens = translation_model.generate(
+                    **inputs, 
+                    forced_bos_token_id=translation_tokenizer.lang_code_to_id[target_code],
+                    max_length=512,
+                    num_beams=4,
+                    early_stopping=True
+                )
+                
+                # Decode output
+                translated_sentence = translation_tokenizer.batch_decode(
+                    translated_tokens, 
+                    skip_special_tokens=True
+                )[0].strip()
+                
+                if translated_sentence:
+                    translated_sentences.append(translated_sentence)
+                else:
+                    translated_sentences.append(f"[Translation failed for: {sentence}]")
+                    
+            except Exception as e:
+                logger.error(f"Error translating sentence {i+1}: {e}")
+                translated_sentences.append(f"[Translation error: {str(e)}]")
+        
+        # Combine translated sentences
+        translated_text = ' '.join(translated_sentences)
+        
+        if not translated_text or translated_text.startswith("[Translation error"):
+            raise HTTPException(status_code=500, detail="Translation returned empty result or encountered errors")
+        
+        # Clean up translation
+        translated_text = re.sub(r'\s+', ' ', translated_text).strip()
+        
+        # Save to cache
+        await save_translation_to_cache(text_hash, target_language_lower, translated_text, "english")
+        
+        return JSONResponse({
+            "file_hash": file_hash,
+            "translated_text": translated_text,
+            "target_language": target_language,
+            "from_cache": False,
+            "source_language": "english",
+            "used_english_reference": True
+        })
+        
+    except Exception as e:
+        logger.error(f"Translation from audio error: {e}")
+        raise HTTPException(status_code=500, detail=f"Translation failed: {str(e)}")
+
+@app.get("/audio_info/{file_hash}")
+async def get_audio_info(file_hash: str):
+    """Get information about cached audio file including English reference availability"""
+    try:
+        with db_manager.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                '''SELECT filename, file_size, language, english_reference, 
+                   created_at, last_accessed FROM cache WHERE file_hash = ?''',
+                (file_hash,)
+            )
+            result = cursor.fetchone()
+            
+            if not result:
+                raise HTTPException(status_code=404, detail="Audio file not found in cache")
+            
+            return JSONResponse({
+                "file_hash": file_hash,
+                "filename": result[0],
+                "file_size": result[1],
+                "detected_language": result[2],
+                "has_english_reference": bool(result[3]),
+                "english_reference_preview": result[3][:100] + "..." if result[3] and len(result[3]) > 100 else result[3],
+                "created_at": result[4],
+                "last_accessed": result[5]
+            })
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error getting audio info: {e}")
+        raise HTTPException(status_code=500, detail="Failed to retrieve audio information")
+
 @app.get("/languages")
 async def get_supported_languages():
     """Get list of supported languages for translation"""
@@ -940,13 +1019,12 @@ async def check_status(file_hash: str):
     cached_result = await get_from_cache(file_hash)
     if cached_result:
         await remove_processing_status(file_hash)
-        cached_data = json.loads(cached_result)
-        cached_data.update({
+        cached_result.update({
             "status": "completed",
             "from_cache": True,
             "message": "Processing completed and result is ready"
         })
-        return JSONResponse(cached_data)
+        return JSONResponse(cached_result)
     
     # Check processing status
     processing_status = await get_processing_status(file_hash)
@@ -978,6 +1056,64 @@ async def health_check():
         "translation_loaded": translation_model is not None
     }
 
+@app.delete("/cache/{file_hash}")
+async def delete_from_cache(file_hash: str):
+    """Delete a specific file from cache"""
+    try:
+        with db_manager.get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute('DELETE FROM cache WHERE file_hash = ?', (file_hash,))
+            cursor.execute('DELETE FROM processing_status WHERE file_hash = ?', (file_hash,))
+            conn.commit()
+            
+            if cursor.rowcount > 0:
+                return JSONResponse({"message": f"File {file_hash} deleted from cache"})
+            else:
+                raise HTTPException(status_code=404, detail="File not found in cache")
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f"Error deleting from cache: {e}")
+        raise HTTPException(status_code=500, detail="Failed to delete from cache")
+
+@app.get("/cache/stats")
+async def get_cache_stats():
+    """Get cache statistics"""
+    try:
+        with db_manager.get_connection() as conn:
+            cursor = conn.cursor()
+            
+            # Cache statistics
+            cursor.execute('SELECT COUNT(*), AVG(file_size) FROM cache')
+            cache_stats = cursor.fetchone()
+            
+            cursor.execute('SELECT COUNT(*) FROM cache WHERE english_reference IS NOT NULL')
+            english_ref_count = cursor.fetchone()[0]
+            
+            cursor.execute('SELECT COUNT(*) FROM translation_cache')
+            translation_count = cursor.fetchone()[0]
+            
+            cursor.execute('''SELECT language, COUNT(*) FROM cache 
+                             WHERE language IS NOT NULL 
+                             GROUP BY language ORDER BY COUNT(*) DESC''')
+            language_stats = cursor.fetchall()
+            
+            cursor.execute('''SELECT target_language, COUNT(*) FROM translation_cache 
+                             GROUP BY target_language ORDER BY COUNT(*) DESC''')
+            translation_stats = cursor.fetchall()
+            
+            return JSONResponse({
+                "total_cached_files": cache_stats[0] or 0,
+                "average_file_size_bytes": int(cache_stats[1] or 0),
+                "files_with_english_reference": english_ref_count,
+                "total_translations": translation_count,
+                "language_distribution": dict(language_stats),
+                "translation_language_distribution": dict(translation_stats)
+            })
+    except Exception as e:
+        logger.error(f"Error getting cache stats: {e}")
+        raise HTTPException(status_code=500, detail="Failed to retrieve cache statistics")
+
 if __name__ == "__main__":
     uvicorn.run(
         app,