",
"Describe Scene": "",
"Detailed Description": "",
"Read Text (OCR)": "",
"Detect Objects": "",
}
TASK_DESCRIPTIONS: Dict[str, str] = {
"Quick Caption": "A brief one-sentence description",
"Describe Scene": "A paragraph describing the scene",
"Detailed Description": "A thorough multi-sentence description",
"Read Text (OCR)": "Reads any visible text aloud",
"Detect Objects": "Names objects and their locations",
}
# ═══════════════════════════════════════════════════════════════
# DEVICE & MODEL LOADING
# ═══════════════════════════════════════════════════════════════
def get_device() -> str:
"""Select best available device."""
if torch.cuda.is_available():
return "cuda"
elif torch.backends.mps.is_available():
return "mps"
return "cpu"
DEVICE: str = get_device()
DTYPE: torch.dtype = torch.float16 if DEVICE == "cuda" else torch.float32
print(f"🖥️ Device: {DEVICE.upper()}")
print(f"🔢 Dtype: {DTYPE}")
# ── Model Loading ──────────────────────────────────────────────
_model_loaded = threading.Event()
processor: Optional[AutoProcessor] = None
model: Optional[AutoModelForCausalLM] = None
def _load_model():
"""Load Florence-2 model in background thread."""
global model, processor
try:
model = AutoModelForCausalLM.from_pretrained(
CONFIG.MODEL_NAME,
trust_remote_code=True,
torch_dtype=DTYPE,
).to(DEVICE).eval()
processor = AutoProcessor.from_pretrained(
CONFIG.MODEL_NAME,
trust_remote_code=True,
)
print("✅ Model loaded successfully")
except Exception as e:
print(f"❌ Model loading failed: {e}")
raise
# Load synchronously on startup (can be made async if needed)
_load_model()
_model_loaded.set()
# ── Background Warmup ──────────────────────────────────────────
_warmup_done = threading.Event()
def _warmup_model():
"""Run a dummy inference to warm up CUDA kernels."""
if model is None or processor is None:
return
try:
dummy = Image.new("RGB", (224, 224), 128)
inputs = processor(text="
", images=dummy, return_tensors="pt").to(DEVICE)
with torch.inference_mode():
model.generate(
input_ids=inputs["input_ids"],
pixel_values=inputs["pixel_values"],
max_new_tokens=10,
num_beams=1,
)
_warmup_done.set()
print("🔥 Model warmed up")
except Exception as e:
print(f"Warmup warning: {e}")
threading.Thread(target=_warmup_model, daemon=True).start()
# ═══════════════════════════════════════════════════════════════
# AUDIO QUEUE SYSTEM
# ═══════════════════════════════════════════════════════════════
class AudioQueue:
"""Thread-safe FIFO audio queue with interruption support."""
def __init__(self, max_size: int = 3):
self._queue: deque[Tuple[str, str]] = deque() # (text, audio_path)
self._current: Optional[str] = None
self._lock = threading.Lock()
self._counter = 0
self._max_size = max_size
def enqueue(self, text: str, audio_path: str) -> Optional[str]:
"""Add audio to queue. Returns the path to play (or None if queue full)."""
with self._lock:
if len(self._queue) >= self._max_size:
# Remove oldest
oldest = self._queue.popleft()
self._safe_delete(oldest[1])
self._queue.append((text, audio_path))
self._counter += 1
return audio_path
def dequeue(self) -> Optional[Tuple[str, str]]:
"""Get next audio item."""
with self._lock:
if self._queue:
item = self._queue.popleft()
self._current = item[1]
return item
return None
def clear(self):
"""Clear all queued audio and delete files."""
with self._lock:
for _, path in self._queue:
self._safe_delete(path)
self._queue.clear()
self._current = None
def interrupt(self):
"""Interrupt current and clear queue."""
self.clear()
@property
def is_empty(self) -> bool:
with self._lock:
return len(self._queue) == 0
@property
def size(self) -> int:
with self._lock:
return len(self._queue)
@staticmethod
def _safe_delete(path: str):
try:
if path and os.path.exists(path):
os.unlink(path)
except OSError:
pass
# Global audio queue
AUDIO_QUEUE = AudioQueue(max_size=CONFIG.MAX_QUEUE_SIZE)
# ═══════════════════════════════════════════════════════════════
# TTS ENGINE (edge-tts)
# ═══════════════════════════════════════════════════════════════
def init_tts_loop() -> asyncio.AbstractEventLoop:
"""Create a dedicated event loop for TTS in a background thread."""
loop = asyncio.new_event_loop()
def _run():
asyncio.set_event_loop(loop)
loop.run_forever()
threading.Thread(target=_run, daemon=True).start()
return loop
_TTS_LOOP = init_tts_loop()
def text_to_speech(text: str, voice_id: str = "en-US-AriaNeural") -> Optional[str]:
"""Convert text to speech, returning the audio file path."""
if not text or not text.strip():
return None
try:
import tempfile
import edge_tts
async def _generate():
with tempfile.NamedTemporaryFile(delete=False, suffix=f".{CONFIG.AUDIO_FORMAT}") as f:
path = f.name
communicate = edge_tts.Communicate(
text.strip(),
voice=voice_id,
rate=CONFIG.TTS_RATE,
)
await communicate.save(path)
return path
future = asyncio.run_coroutine_threadsafe(_generate(), _TTS_LOOP)
return future.result(timeout=CONFIG.TTS_TIMEOUT)
except Exception as e:
print(f"TTS error: {e}")
return None
# ═══════════════════════════════════════════════════════════════
# APPLICATION STATE
# ═══════════════════════════════════════════════════════════════
@dataclass
class AppState:
"""Thread-safe application state."""
# Scene hashing
last_hash: Optional[bytes] = None
last_task: str = ""
last_text: str = ""
last_audio: Optional[str] = None
# Realtime
realtime_active: bool = False
last_capture_time: float = 0.0
# History
history: List[Dict[str, Any]] = field(default_factory=list)
max_history: int = 50
# Stats
total_describes: int = 0
total_realtime_captures: int = 0
# Lock
_lock: threading.Lock = field(default_factory=threading.Lock)
_tmp_files: List[Tuple[str, float]] = field(default_factory=list)
def update(self, hash_val: bytes, task: str, text: str, audio: Optional[str]):
"""Update state with new capture results."""
with self._lock:
self._cleanup_old_files()
if self.last_audio and os.path.exists(self.last_audio):
try:
os.unlink(self.last_audio)
except OSError:
pass
self.last_hash = hash_val
self.last_task = task
self.last_text = text
self.last_audio = audio
if audio:
self._tmp_files.append((audio, time.time()))
# Add to history
self.history.insert(0, {
"time": time.strftime("%H:%M:%S"),
"task": task,
"text": text,
})
if len(self.history) > self.max_history:
self.history = self.history[: self.max_history]
def is_duplicate(self, hash_val: bytes, task: str) -> bool:
"""Check if this hash+task combination was already processed."""
with self._lock:
return (
self.last_hash is not None
and self.last_hash == hash_val
and self.last_task == task
and self.last_text != ""
)
def get_last(self) -> Tuple[str, Optional[str]]:
"""Get last description text and audio."""
with self._lock:
return self.last_text, self.last_audio
def add_stat(self, key: str):
with self._lock:
if key == "describe":
self.total_describes += 1
elif key == "realtime":
self.total_realtime_captures += 1
def get_stats(self) -> Dict[str, Any]:
with self._lock:
return {
"describes": self.total_describes,
"realtime_captures": self.total_realtime_captures,
"history_count": len(self.history),
}
def _cleanup_old_files(self):
"""Remove temp files older than 5 minutes."""
now = time.time()
keep = []
for path, ts in self._tmp_files:
if now - ts > 300: # 5 minutes
try:
os.unlink(path)
except OSError:
pass
else:
keep.append((path, ts))
self._tmp_files = keep
# Global state
APP_STATE = AppState()
# ═══════════════════════════════════════════════════════════════
# IMAGE PROCESSING
# ═══════════════════════════════════════════════════════════════
def compute_hash(image: Image.Image, size: int = 16) -> bytes:
"""Compute difference hash (dHash) for scene change detection."""
gray = image.resize((size + 1, size), Image.LANCZOS).convert("L")
pixels = list(gray.getdata())
return bytes(
1 if pixels[y * (size + 1) + x] > pixels[y * (size + 1) + x + 1] else 0
for y in range(size)
for x in range(size)
)
def hash_distance(a: Optional[bytes], b: Optional[bytes]) -> float:
"""Compute normalized Hamming distance between two hashes."""
if a is None or b is None:
return 1.0
if len(a) != len(b):
return 1.0
return sum(x != y for x, y in zip(a, b)) / len(a)
def preprocess_image(image: Image.Image) -> Image.Image:
"""Resize image for inference while preserving aspect ratio."""
w, h = image.size
if max(w, h) <= CONFIG.MAX_DIM:
return image
scale = CONFIG.MAX_DIM / max(w, h)
new_size = (int(w * scale), int(h * scale))
return image.resize(new_size, Image.LANCZOS)
def auto_enhance(image: Image.Image) -> Image.Image:
"""Auto-enhance image for better vision model performance."""
# Slight contrast boost helps Florence-2 on low-light images
enhancer = ImageEnhance.Contrast(image)
image = enhancer.enhance(1.1)
return image
# ═══════════════════════════════════════════════════════════════
# CORE VISION INFERENCE
# ═══════════════════════════════════════════════════════════════
def run_inference(image: Image.Image, task_label: str) -> str:
"""Run Florence-2 inference on an image."""
if model is None or processor is None:
return "Error: Model not loaded. Please wait or restart."
task_token = TASKS.get(task_label, "
")
max_tokens = CONFIG.MAX_NEW_TOKENS.get(task_token, 64)
try:
# Preprocess
image = preprocess_image(image)
image = auto_enhance(image)
# Prepare inputs
inputs = processor(
text=task_token,
images=image,
return_tensors="pt",
).to(DEVICE)
# Generate
with torch.inference_mode():
output_ids = model.generate(
input_ids=inputs["input_ids"],
pixel_values=inputs["pixel_values"],
max_new_tokens=max_tokens,
do_sample=False,
num_beams=1,
use_cache=True,
)
# Decode
raw_text = processor.batch_decode(output_ids, skip_special_tokens=False)[0]
result = processor.post_process_generation(
raw_text,
task=task_token,
image_size=(image.width, image.height),
)
# Format output based on task
if task_token == "":
od_data = result.get("", {})
return format_object_detection(od_data)
elif task_token == "":
text_found = result.get("", "").strip()
if not text_found:
return "No text detected in the image."
return f"Text found: {text_found}"
else:
caption = result.get(task_token, "").strip()
if not caption:
return "I couldn't understand what's in the image. Please try again."
return caption
except torch.cuda.OutOfMemoryError:
torch.cuda.empty_cache()
return "The image is too large for memory. Try a smaller image."
except Exception as e:
print(f"Inference error: {e}")
return f"Sorry, I had trouble analyzing that image. Please try again."
def format_object_detection(od_data: Dict) -> str:
"""Format object detection results into natural language."""
if not od_data or not od_data.get("labels"):
return "No objects detected in the image."
labels = od_data.get("labels", [])
bboxes = od_data.get("bboxes", [])
if not labels:
return "No objects detected in the image."
# Build object list with positions
objects: List[Tuple[str, str]] = []
for label, bbox in zip(labels, bboxes):
x1, _, x2, _ = bbox
cx = (x1 + x2) / 2
# Florence uses 0-999 coordinate space
if cx < 333:
pos = "on the left"
elif cx < 666:
pos = "in the center"
else:
pos = "on the right"
objects.append((label.strip(), pos))
# Deduplicate (keep first occurrence of each label type)
seen: set = set()
unique: List[Tuple[str, str]] = []
for lbl, pos in objects:
key = lbl.lower()
if key and key not in seen:
seen.add(key)
unique.append((lbl, pos))
if not unique:
return "No objects detected in the image."
# Format naturally
if len(unique) == 1:
lbl, pos = unique[0]
return f"I see {lbl} {pos}."
parts = [f"{lbl} {pos}" for lbl, pos in unique]
if len(parts) <= 5:
return "I see " + ", ".join(parts[:-1]) + f", and {parts[-1]}."
else:
summary = ", ".join(parts[:5])
return f"I see {len(unique)} objects: {summary}, and {len(unique) - 5} more."
# ═══════════════════════════════════════════════════════════════
# HANDLER FUNCTIONS
# ═══════════════════════════════════════════════════════════════
def _status_html(msg: str) -> str:
"""Wrap a status message in the styled status-bar div."""
return f'
{msg}
'
def describe_now(image, task_label: str, voice_name: str):
"""
Manual describe handler.
Streams words visually, then returns final text + audio.
"""
if image is None:
yield "📷 Please open the camera or upload an image first.", None, _status_html("Waiting for image...")
return
# Convert to PIL if needed
if not isinstance(image, Image.Image):
image = Image.fromarray(image)
# Compute hash
img_hash = compute_hash(image)
task_key = TASKS.get(task_label, "
")
# Check cache
if APP_STATE.is_duplicate(img_hash, task_key):
text, audio = APP_STATE.get_last()
yield text, audio, _status_html(f"✓ Cached result • {task_label}")
return
# Run inference
yield "⏳ Analyzing image...", None, _status_html("⏳ Processing image...")
caption = run_inference(image, task_label)
APP_STATE.add_stat("describe")
# Stream words
words = caption.split()
partial = ""
for i, w in enumerate(words):
partial += (" " if partial else "") + w
if (i + 1) % 3 == 0 or i == len(words) - 1:
yield partial, None, _status_html(f"⏳ Generating description... ({i + 1}/{len(words)} words)")
# Generate TTS
voice_id = VOICE_MAP.get(voice_name, "en-US-AriaNeural")
audio_path = text_to_speech(caption, voice_id)
# Update state
APP_STATE.update(img_hash, task_key, caption, audio_path)
yield caption, audio_path, _status_html(f"✅ {task_label} complete • {len(words)} words")
def handle_upload(image, task_label: str, voice_name: str):
"""Handle uploaded image — always processes fresh, bypasses scene cache."""
if image is None:
yield "📁 Please upload an image first.", None, _status_html("Waiting for upload...")
return
# Convert to PIL if needed
if not isinstance(image, Image.Image):
image = Image.fromarray(image)
yield "⏳ Analyzing uploaded image...", None, _status_html("⏳ Processing uploaded image...")
caption = run_inference(image, task_label)
APP_STATE.add_stat("describe")
# Stream words visually
words = caption.split()
partial = ""
for i, w in enumerate(words):
partial += (" " if partial else "") + w
if (i + 1) % 3 == 0 or i == len(words) - 1:
yield partial, None, _status_html(f"⏳ Generating description... ({i + 1}/{len(words)} words)")
# Generate TTS
voice_id = VOICE_MAP.get(voice_name, "en-US-AriaNeural")
audio_path = text_to_speech(caption, voice_id)
# Update app state
img_hash = compute_hash(image)
APP_STATE.update(img_hash, TASKS.get(task_label, "
"), caption, audio_path)
yield caption, audio_path, _status_html(f"✅ Upload — {task_label} complete • {len(words)} words")
def handle_realtime_stream(image, task_label: str, voice_name: str, rt_active: bool):
"""
Called automatically by webcam.stream() every CAPTURE_INTERVAL seconds.
Only processes if realtime toggle is ON.
"""
if not rt_active:
return gr.update(), gr.update(), _status_html("⚫ Realtime paused — press R to start")
if image is None:
return gr.update(), gr.update(), _status_html("📷 No camera feed detected")
# Convert to PIL
if not isinstance(image, Image.Image):
image = Image.fromarray(image)
# Debounce check
now = time.time()
if now - APP_STATE.last_capture_time < 1.0:
return gr.update(), gr.update(), _status_html("⏳ Debouncing...")
APP_STATE.last_capture_time = now
# Compute hash
img_hash = compute_hash(image)
task_key = TASKS.get(task_label, "
'
"✅ Ready — Press D to describe what the camera sees"
"
"
)
# ── Header ───────────────────────────────────────────
gr.Markdown(
f"# 👁️ {CONFIG.APP_NAME} — Realtime Vision Assistant",
elem_classes=["echo-section-title"],
)
gr.Markdown(
"Helping blind and visually impaired users understand their surroundings. "
"Press **D** to describe, **R** for realtime mode, **P** to repeat."
)
# ── Accessibility Toolbar ────────────────────────────
gr.HTML("""
Text Size:D describe ·
R realtime ·
P repeat ·
Esc stop
"""
)
# ── Realtime state (single source of truth) ──────────
rt_state = gr.State(False)
with gr.Row():
# ════════════════════════════════════════════════
# LEFT COLUMN — Inputs
# ════════════════════════════════════════════════
with gr.Column(scale=1):
# ── Camera ─────────────────────────────────
webcam = gr.Image(
label="📷 Camera Feed",
type="numpy",
sources=["webcam"],
streaming=True,
height=260,
elem_id="echo-webcam",
)
# ── Upload ─────────────────────────────────
upload = gr.Image(
label="📁 Or Upload Image",
type="numpy",
sources=["upload"],
height=140,
elem_id="echo-upload",
)
# ── Task Selection ─────────────────────────
task_radio = gr.Radio(
choices=list(TASKS.keys()),
value="Quick Caption",
label="What should I do?",
info="Select the type of description you want",
)
# Task description
task_info = gr.Textbox(
value=TASK_DESCRIPTIONS["Quick Caption"],
label="",
interactive=False,
max_lines=1,
show_label=False,
container=False,
elem_classes=["echo-stats"],
)
# ── Voice Selection ────────────────────────
voice_dropdown = gr.Dropdown(
choices=list(VOICE_MAP.keys()),
value="Aria — Female US",
label="🔊 Voice",
info="Choose a voice for spoken descriptions",
)
# ── Describe Button ────────────────────────
describe_btn = gr.Button(
"🔍 Describe Now (D)",
variant="primary",
size="lg",
elem_id="echo-describe-btn",
)
# ── Realtime Toggle ────────────────────────
realtime_btn = gr.Button(
"⚫ Start Realtime (R)",
variant="secondary",
size="lg",
elem_id="echo-rt-btn",
)
# ════════════════════════════════════════════════
# RIGHT COLUMN — Output
# ════════════════════════════════════════════════
with gr.Column(scale=1):
# ── Caption Output ─────────────────────────
caption_box = gr.Textbox(
label="📝 Description",
lines=6,
interactive=False,
show_copy_button=True,
placeholder="Description will appear here...",
elem_id="echo-caption",
)
# ── Audio Output ───────────────────────────
audio_player = gr.Audio(
label="🔊 Audio",
type="filepath",
autoplay=True,
elem_id="echo-audio",
)
# ── Action Buttons ─────────────────────────
with gr.Row():
repeat_btn = gr.Button(
"🔁 Repeat Last (P)",
variant="secondary",
size="lg",
elem_id="echo-repeat-btn",
)
stop_btn = gr.Button(
"⏹ Stop All (Esc)",
variant="stop",
size="lg",
elem_id="echo-stop-btn",
)
# ── Tips ───────────────────────────────────
gr.HTML("""
💡 Tips:
• D — Describe what the camera sees right now
• R — Start/stop auto-description every 3 seconds
• P — Repeat the last description
• Esc — Stop all audio and realtime mode
• Read Text — Reads signs, labels, screens (OCR)
• Detect Objects — Hear what's where in the scene