iris-visual-assistant / priority_engine.py
Robin7339's picture
Upload 4 files
cb6be11 verified
Raw
History Blame Contribute Delete
3.87 kB
"""
IRIS - Priority / Context Engine
Takes YOLO detections + optional VLM reasoning output and selects
EXACTLY ONE instruction to speak. Never a list β€” always the single
most relevant thing for the visually impaired user right now.
Priority order:
1. VLM reasoning output (context-aware, if available)
2. Urgent object blocking center path (person, car, etc.)
3. Caution object in center (chair, bench, stairs, etc.)
4. Closest high-conf object on left or right
5. General scene clear message
"""
class PriorityEngine:
"""
Selects exactly one navigation instruction from structured detections
and optional VLM context reasoning.
"""
# Objects that trigger immediate caution warnings
URGENT = {
"person", "car", "truck", "bus", "motorcycle", "bicycle",
"dog", "cat", "horse", "traffic light", "stop sign",
}
# Objects that need caution but are less mobile
CAUTION = {
"chair", "bench", "dining table", "potted plant", "suitcase",
"backpack", "umbrella", "fire hydrant", "parking meter",
"stairs", "step", "pole", "bollard",
}
# Position β†’ spoken phrase
POS_PHRASE = {
"left": "on your left",
"center": "directly ahead",
"right": "on your right",
}
def pick(self, detections: list, vlm_text: str = "") -> str:
"""
Return exactly ONE instruction string.
Args:
detections: sorted YOLO detections (highest confidence first)
vlm_text: reasoning from VLM engine (empty string if unavailable)
Returns:
A single short instruction for TTS.
"""
# ── 1. VLM reasoning takes highest priority (context-aware) ──────────
if vlm_text and len(vlm_text.strip()) > 5:
return self._clean(vlm_text)
if not detections:
return "Path ahead looks clear."
# ── 2. Urgent object directly ahead ──────────────────────────────────
center_urgent = [
d for d in detections
if d["position"] == "center" and d["object"] in self.URGENT
]
if center_urgent:
obj = center_urgent[0]["object"]
return f"Caution! {obj.capitalize()} directly ahead."
# ── 3. Any object blocking center ────────────────────────────────────
center_any = [d for d in detections if d["position"] == "center"]
if center_any:
obj = center_any[0]["object"]
if obj in self.CAUTION:
return f"Watch out β€” {obj} ahead. Step around it."
return f"{obj.capitalize()} ahead. Proceed carefully."
# ── 4. Urgent object on sides ────────────────────────────────────────
side_urgent = [
d for d in detections
if d["position"] in ("left", "right") and d["object"] in self.URGENT
]
if side_urgent:
d = side_urgent[0]
pos = self.POS_PHRASE.get(d["position"], d["position"])
return f"{d['object'].capitalize()} {pos}. Stay aware."
# ── 5. Highest confidence detection anywhere ──────────────────────────
top = detections[0]
pos = self.POS_PHRASE.get(top["position"], top["position"])
return f"{top['object'].capitalize()} {pos}."
@staticmethod
def _clean(text: str) -> str:
"""Ensure sentence ends with a period and is clean."""
text = text.strip()
if text and not text.endswith((".", "!", "?")):
text += "."
return text