import gradio as gr import easyocr from deep_translator import GoogleTranslator from PIL import Image, ImageDraw, ImageFont import numpy as np import cv2 import time import re from typing import Tuple, List, Optional, Dict import io import os from collections import defaultdict import math # Global variables reader = None translation_cache = {} # Define supported languages SUPPORTED_LANGUAGES = { 'en': 'English', 'hi': 'Hindi', 'es': 'Spanish', 'fr': 'French', 'de': 'German', 'ja': 'Japanese', 'ko': 'Korean', 'zh': 'Chinese' } # Language code mapping for Google Translator LANG_CODE_MAP = { 'English': 'en', 'Hindi': 'hi', 'Spanish': 'es', 'French': 'fr', 'German': 'de', 'Japanese': 'ja', 'Korean': 'ko', 'Chinese': 'zh' } def initialize_reader(): """Initialize EasyOCR reader with fallback options""" global reader if reader is None: # Try different initialization strategies init_strategies = [ (['en', 'hi'], "English and Hindi"), (['en'], "English only"), (['en', 'hi'], "English and Hindi with verbose"), ] for i, (languages, description) in enumerate(init_strategies): try: print(f"Attempting OCR initialization: {description}") verbose_setting = True if i == 2 else False reader = easyocr.Reader( languages, gpu=False, verbose=verbose_setting, download_enabled=True, detector=True, recognizer=True ) print(f"✅ EasyOCR initialized successfully with {description}") return reader except ImportError as e: print(f"❌ Import error: {e}") continue except Exception as e: print(f"❌ Initialization attempt {i+1} failed: {e}") if i < len(init_strategies) - 1: print("Trying alternative approach...") continue else: print("All initialization strategies failed") # If all strategies fail, return None reader = None print("❌ Could not initialize EasyOCR with any strategy") return reader def calculate_distance(box1, box2): """Calculate distance between two bounding boxes""" # Get center points center1 = [(box1[0][0] + box1[2][0]) / 2, (box1[0][1] + box1[2][1]) / 2] center2 = [(box2[0][0] + box2[2][0]) / 2, (box2[0][1] + box2[2][1]) / 2] return math.sqrt((center1[0] - center2[0])**2 + (center1[1] - center2[1])**2) def are_boxes_on_same_line(box1, box2, tolerance=20): """Check if two bounding boxes are on the same horizontal line""" # Get y-coordinates (vertical positions) y1_avg = (box1[0][1] + box1[2][1]) / 2 y2_avg = (box2[0][1] + box2[2][1]) / 2 return abs(y1_avg - y2_avg) <= tolerance def group_text_regions(ocr_results, line_tolerance=25, proximity_threshold=50): """Group OCR results into meaningful text blocks""" if not ocr_results: return [] # Sort by vertical position first, then horizontal sorted_results = sorted(ocr_results, key=lambda x: (x[0][0][1], x[0][0][0])) grouped_lines = [] current_line = [sorted_results[0]] for i in range(1, len(sorted_results)): current_box = sorted_results[i][0] prev_box = current_line[-1][0] # Check if boxes are on the same line if are_boxes_on_same_line(current_box, prev_box, line_tolerance): # Check proximity (not too far apart horizontally) if calculate_distance(current_box, prev_box) <= proximity_threshold: current_line.append(sorted_results[i]) else: # Start new line if too far apart grouped_lines.append(current_line) current_line = [sorted_results[i]] else: # Different line grouped_lines.append(current_line) current_line = [sorted_results[i]] # Don't forget the last line if current_line: grouped_lines.append(current_line) # Merge text within each line merged_groups = [] for line in grouped_lines: if len(line) == 1: merged_groups.append(line[0]) else: # Sort by horizontal position within the line line.sort(key=lambda x: x[0][0][0]) # Merge text merged_text = ' '.join([item[1] for item in line]) # Create combined bounding box all_points = [] for item in line: all_points.extend(item[0]) # Find min/max coordinates x_coords = [point[0] for point in all_points] y_coords = [point[1] for point in all_points] min_x, max_x = min(x_coords), max(x_coords) min_y, max_y = min(y_coords), max(y_coords) # Create new bounding box merged_bbox = [[min_x, min_y], [max_x, min_y], [max_x, max_y], [min_x, max_y]] # Use average confidence avg_confidence = sum([item[2] for item in line]) / len(line) merged_groups.append((merged_bbox, merged_text, avg_confidence)) return merged_groups def get_font_for_text(text: str, target_size: int = 20) -> ImageFont.FreeTypeFont: """Get appropriate font based on text content""" # Check for different scripts has_devanagari = bool(re.search(r'[\u0900-\u097F]', text)) has_chinese = bool(re.search(r'[\u4e00-\u9fff]', text)) has_japanese = bool(re.search(r'[\u3040-\u309f\u30a0-\u30ff]', text)) has_korean = bool(re.search(r'[\uac00-\ud7af]', text)) has_arabic = bool(re.search(r'[\u0600-\u06ff]', text)) # Font paths for different scripts font_paths = [] if has_devanagari: font_paths.extend([ "/usr/share/fonts/truetype/noto/NotoSansDevanagari-Regular.ttf", "/usr/share/fonts/truetype/lohit-devanagari/Lohit-Devanagari.ttf" ]) if has_chinese or has_japanese: font_paths.extend([ "/usr/share/fonts/truetype/noto/NotoSansCJK-Regular.ttc", "/usr/share/fonts/truetype/arphic/uming.ttc" ]) if has_korean: font_paths.append("/usr/share/fonts/truetype/noto/NotoSansKR-Regular.otf") if has_arabic: font_paths.append("/usr/share/fonts/truetype/noto/NotoSansArabic-Regular.ttf") # Default fonts font_paths.extend([ "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", "/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf", "/usr/share/fonts/truetype/noto/NotoSans-Bold.ttf" ]) for font_path in font_paths: try: if os.path.exists(font_path): return ImageFont.truetype(font_path, size=target_size) except (OSError, IOError): continue # Fallback try: return ImageFont.load_default() except: return None def smart_translate_with_context(text: str, target_lang: str, source_lang: str = 'auto') -> str: """Enhanced translation with better context handling""" if not text or not text.strip(): return "" # Clean and normalize text cleaned_text = re.sub(r'\s+', ' ', text.strip()) # Cache key cache_key = f"{cleaned_text}|{source_lang}|{target_lang}" if cache_key in translation_cache: return translation_cache[cache_key] # Pre-processing for better translation context # Handle common signboard patterns signboard_patterns = { r'\b(no|not|don\'t|do not)\s+(use|mobile|phone|cell)\b': 'prohibition_mobile', r'\b(please|kindly)\s+(do not|don\'t)\s+(use|mobile|phone)\b': 'polite_prohibition_mobile', r'\b(exit|entrance|entry|way out|way in)\b': 'direction', r'\b(toilet|restroom|bathroom|washroom)\b': 'facility', r'\b(parking|park|no parking)\b': 'parking', r'\b(emergency|fire|safety)\b': 'safety' } context_hint = "" for pattern, context in signboard_patterns.items(): if re.search(pattern, cleaned_text.lower()): context_hint = f"[Signboard context: {context}] " break max_retries = 3 for attempt in range(max_retries): try: translator = GoogleTranslator(source=source_lang, target=target_lang) # Add context hint for better translation text_to_translate = context_hint + cleaned_text if context_hint else cleaned_text translated = translator.translate(text_to_translate) if translated and translated.strip(): # Remove context hint from result if it was added if context_hint and translated.startswith('['): # Try to remove the context hint from translation bracket_end = translated.find('] ') if bracket_end != -1: translated = translated[bracket_end + 2:].strip() # Post-process for common improvements translated = post_process_translation(translated, target_lang) # Cache successful translation translation_cache[cache_key] = translated return translated except Exception as e: print(f"Translation attempt {attempt + 1} failed: {e}") if attempt < max_retries - 1: time.sleep(0.5) return cleaned_text def post_process_translation(translated_text: str, target_lang: str) -> str: """Post-process translation for better quality""" # Language-specific post-processing if target_lang == 'hi': # Hindi # Common corrections for Hindi translations corrections = { 'मत करो': 'न करें', # More polite form 'का उपयोग मत करो': 'का उपयोग न करें', 'फोन का उपयोग': 'मोबाइल का उपयोग' } for old, new in corrections.items(): translated_text = translated_text.replace(old, new) return translated_text.strip() def calculate_optimal_font_size(text: str, bbox_width: int, bbox_height: int, min_size: int = 12, max_size: int = 48) -> int: """Calculate optimal font size with better scaling""" if not text: return min_size # Estimate character width (varies by language) char_width_ratio = 0.7 # More conservative estimate # For non-Latin scripts, adjust ratio if re.search(r'[\u0900-\u097F\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af]', text): char_width_ratio = 0.9 # Wider characters # Calculate based on width constraint width_based_size = int(bbox_width / (len(text) * char_width_ratio)) # Calculate based on height constraint (use 80% of available height) height_based_size = int(bbox_height * 0.8) # Take the smaller constraint optimal_size = min(width_based_size, height_based_size) # Apply bounds return max(min_size, min(optimal_size, max_size)) def get_contrasting_color(bg_color: Tuple[int, int, int]) -> Tuple[int, int, int]: """Get contrasting text color""" r, g, b = bg_color[:3] luminance = (0.299 * r + 0.587 * g + 0.114 * b) / 255 if luminance > 0.5: return (0, 0, 0) # Black text for light background else: return (255, 255, 255) # White text for dark background def extract_dominant_color(image: np.ndarray, bbox: List) -> Tuple[int, int, int]: """Extract dominant color from the bounding box region""" try: # Get bounding box coordinates points = np.array(bbox, dtype=np.int32) # Create mask for the region mask = np.zeros(image.shape[:2], dtype=np.uint8) cv2.fillPoly(mask, [points], 255) # Extract pixels within the region region_pixels = image[mask > 0] if len(region_pixels) > 0: # Calculate mean color mean_color = np.mean(region_pixels, axis=0) return tuple(map(int, mean_color)) except Exception as e: print(f"Error extracting color: {e}") return (240, 240, 240) # Default light gray def create_enhanced_overlay(image: Image.Image, bbox: List, translated_text: str, bg_opacity: int = 180): """Create enhanced overlay with better positioning""" draw = ImageDraw.Draw(image, 'RGBA') # Convert bbox to integer coordinates points = [[int(p[0]), int(p[1])] for p in bbox] # Calculate bounding rectangle x_coords = [p[0] for p in points] y_coords = [p[1] for p in points] x_min, x_max = min(x_coords), max(x_coords) y_min, y_max = min(y_coords), max(y_coords) width = x_max - x_min height = y_max - y_min # Calculate optimal font size font_size = calculate_optimal_font_size(translated_text, width, height) font = get_font_for_text(translated_text, font_size) # Extract background color img_array = np.array(image.convert('RGB')) bg_color = extract_dominant_color(img_array, bbox) # Create semi-transparent background padding = max(4, font_size // 6) bg_rect = [ x_min - padding, y_min - padding, x_max + padding, y_max + padding ] # Draw background with original color but semi-transparent bg_color_with_alpha = bg_color + (bg_opacity,) draw.rectangle(bg_rect, fill=bg_color_with_alpha) # Calculate text position (center alignment) try: bbox_text = draw.textbbox((0, 0), translated_text, font=font) text_width = bbox_text[2] - bbox_text[0] text_height = bbox_text[3] - bbox_text[1] except: text_width = len(translated_text) * font_size * 0.6 text_height = font_size text_x = x_min + (width - text_width) / 2 text_y = y_min + (height - text_height) / 2 # Get contrasting text color text_color = get_contrasting_color(bg_color) # Draw text with slight shadow for better readability shadow_offset = max(1, font_size // 20) shadow_color = (0, 0, 0) if text_color == (255, 255, 255) else (255, 255, 255) # Draw shadow draw.text((text_x + shadow_offset, text_y + shadow_offset), translated_text, fill=shadow_color + (100,), font=font) # Draw main text draw.text((text_x, text_y), translated_text, fill=text_color, font=font) def process_image_enhanced(image: Image.Image, target_language: str, progress=gr.Progress()) -> Tuple[Optional[Image.Image], str]: """Enhanced image processing with better text grouping""" if image is None: return None, "❌ Please upload an image first." if target_language not in LANG_CODE_MAP: return image, f"❌ Unsupported target language: {target_language}" target_lang_code = LANG_CODE_MAP[target_language] progress(0.1, "🔧 Initializing OCR engine...") # Initialize OCR with better error handling try: ocr = initialize_reader() if ocr is None: return image, """❌ OCR initialization failed. This might be due to: • Missing system dependencies • Network issues downloading models • Insufficient memory Please try refreshing the page or contact support.""" # Test OCR with a simple operation test_array = np.array(image.convert('RGB')) if test_array.size == 0: return image, "❌ Invalid image format. Please upload a valid image file." except Exception as e: error_details = str(e) return image, f"""❌ OCR Setup Error: {error_details} Possible solutions: • Refresh the browser and try again • Upload a different image format (JPG/PNG) • Check if the image is not corrupted Technical details: {type(e).__name__}""" progress(0.3, "🔍 Extracting and grouping text regions...") try: # Convert PIL image to numpy array with error handling img_array = np.array(image.convert('RGB')) if img_array is None or img_array.size == 0: return image, "❌ Error processing image. Please try a different image." print(f"Image shape: {img_array.shape}") # Perform OCR with error handling and fallback options try: results = ocr.readtext(img_array, detail=1, paragraph=False, width_ths=0.7, height_ths=0.7) except Exception as ocr_error: print(f"Primary OCR failed: {ocr_error}") # Fallback: try with different parameters try: results = ocr.readtext(img_array, detail=1) except Exception as fallback_error: print(f"Fallback OCR failed: {fallback_error}") return image, f"""❌ OCR Processing Failed: {str(ocr_error)} Troubleshooting: • Image might be too complex or low quality • Try uploading a clearer image • Ensure text is clearly visible Fallback error: {str(fallback_error)}""" if not results: return image, """ℹ️ No readable text found in the image. Tips for better results: • Ensure text is clearly visible and well-lit • Upload higher resolution images • Make sure text is not too small or blurry""" # Filter by confidence filtered_results = [(bbox, text, conf) for bbox, text, conf in results if conf > 0.4 and text.strip()] if not filtered_results: return image, "ℹ️ No text detected with sufficient confidence." progress(0.5, "🔗 Grouping related text regions...") # Group text regions for contextual translation grouped_results = group_text_regions(filtered_results) progress(0.6, f"🌐 Translating {len(grouped_results)} text groups...") # Create result image result_image = image.copy().convert('RGBA') translation_info = [] for i, (bbox, text, confidence) in enumerate(grouped_results): progress(0.6 + (0.3 * i / len(grouped_results)), f"Translating group {i+1}/{len(grouped_results)}") if text and text.strip(): # Clean text cleaned_text = re.sub(r'\s+', ' ', text.strip()) # Translate with context translated = smart_translate_with_context(cleaned_text, target_lang_code) # Create overlay create_enhanced_overlay(result_image, bbox, translated) # Store info translation_info.append({ 'original': cleaned_text, 'translated': translated, 'confidence': confidence }) progress(1.0, "✅ Translation completed!") # Convert to RGB final_image = result_image.convert('RGB') # Create detailed summary summary_lines = [f"🎯 Successfully processed {len(translation_info)} text groups:\n"] for i, info in enumerate(translation_info, 1): summary_lines.append(f"**Group {i}:**") summary_lines.append(f"📝 Original: _{info['original']}_") summary_lines.append(f"🌐 Translation: **{info['translated']}**") summary_lines.append(f"📊 Confidence: {info['confidence']:.2f}") summary_lines.append("") summary_text = "\n".join(summary_lines) return final_image, summary_text except Exception as e: error_msg = f"❌ Error processing image: {str(e)}" print(f"Processing error: {e}") return image, error_msg # Enhanced CSS custom_css = """ .gradio-container { max-width: 1400px; margin: auto; font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; } .main-header { text-align: center; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text; font-size: 2.8em; font-weight: 800; margin-bottom: 0.5em; text-shadow: 2px 2px 4px rgba(0,0,0,0.1); } .description { text-align: center; font-size: 1.2em; color: #555; margin-bottom: 2em; line-height: 1.6; } .feature-box { background: linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%); padding: 1.5em; border-radius: 12px; margin: 1.5em 0; box-shadow: 0 4px 6px rgba(0,0,0,0.1); } .improvement-box { background: linear-gradient(135deg, #a8edea 0%, #fed6e3 100%); padding: 1.2em; border-radius: 10px; margin: 1em 0; border-left: 4px solid #667eea; } .btn-primary { background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border: none; font-weight: 600; text-transform: uppercase; letter-spacing: 1px; } """ # Create Gradio interface with gr.Blocks(css=custom_css, title="Enhanced Multilingual Signboard Translator") as demo: gr.HTML("""
🌐 Enhanced Multilingual Signboard Translator
Advanced OCR with intelligent text grouping and contextual translation overlay
""") with gr.Row(): with gr.Column(scale=1): gr.Markdown("### 📤 Upload & Configure") input_image = gr.Image( label="📷 Upload Signboard Image", type="pil", height=350 ) target_language = gr.Dropdown( choices=list(LANG_CODE_MAP.keys()), value="Hindi", label="🎯 Target Language", info="Select language for translation" ) translate_btn = gr.Button( "🚀 Translate Signboard", variant="primary", size="lg", elem_classes=["btn-primary"] ) with gr.Column(scale=1): gr.Markdown("### 📋 Results") output_image = gr.Image( label="🖼️ Translated Signboard", type="pil", height=350 ) output_text = gr.Textbox( label="📝 Translation Analysis", lines=10, max_lines=20, info="Detailed breakdown of detected and translated text" ) # Event binding translate_btn.click( fn=process_image_enhanced, inputs=[input_image, target_language], outputs=[output_image, output_text], show_progress=True ) # Enhanced information sections gr.HTML("""

🚀 Key Improvements in This Version:

""") gr.HTML("""

✨ Advanced Features:

""") if __name__ == "__main__": print("🔧 Initializing Enhanced OCR Translator...") print("System Information:") print(f"Python version: {os.sys.version}") print(f"NumPy version: {np.__version__}") # Pre-initialize with detailed logging try: print("Starting OCR initialization...") ocr_reader = initialize_reader() if ocr_reader: print("✅ OCR System ready!") else: print("⚠️ OCR initialization failed - will retry when needed") except Exception as e: print(f"⚠️ Pre-initialization error: {e}") print("OCR will be initialized on first use") # Launch with better error handling try: demo.launch( share=True, show_error=True, server_name="0.0.0.0", server_port=7860, enable_queue=True ) except Exception as e: print(f"Launch error: {e}") # Fallback launch demo.launch()