apps/api/app/lenses/object_grounding.py

from __future__ import annotations

from app.providers.vision.base import VisionProvider
from app.storage.models import VisualAssetRecord

from .base import Lens


class ObjectGroundingLens(Lens):
    name = "object_grounding"
    view_type = "objects"

    def __init__(self, provider: VisionProvider):
        self.provider = provider

    async def execute(self, asset: VisualAssetRecord, image_bytes: bytes, parameters: dict) -> tuple[dict, float, list[dict]]:
        phrase = parameters.get("phrase", "object")
        payload = await self.provider.ground_object(image_bytes, phrase)
        objects = payload.get("objects", [])
        confidence = max([o.get("confidence", 0.0) for o in objects] or [0.0])
        provenance = [
            {
                "asset_id": asset.id,
                "bbox": obj.get("bbox"),
                "extraction_method": self.name,
                "confidence": obj.get("confidence", confidence),
                "lens_version": self.version,
            }
            for obj in objects
        ]
        return payload, confidence, provenance