import torch from transformers import AutoModelForCausalLM, AutoProcessor from PIL import Image from typing import Optional, Dict, Any from src.vision.vision_engine import VisionEngine from src.config import CONFIG from src.vision.utils import preprocess_image, auto_enhance from src.vision.captioning import format_caption from src.vision.ocr import format_ocr from src.vision.detection import format_object_detection try: import spaces IS_SPACES = True gpu_decorator = spaces.GPU except ImportError: IS_SPACES = False def gpu_decorator(func): return func def get_device() -> str: """Select best available device.""" if torch.cuda.is_available(): return "cuda" elif torch.backends.mps.is_available(): return "mps" return "cpu" DEVICE: str = "cpu" if IS_SPACES else get_device() DTYPE: torch.dtype = torch.float16 if IS_SPACES or DEVICE == "cuda" else torch.float32 class FlorenceVisionEngine(VisionEngine): def __init__(self): self.model: Optional[AutoModelForCausalLM] = None self.processor: Optional[AutoProcessor] = None self.paddle_ocr = None def load(self): """Load the Florence-2 model.""" if self.model is not None: return try: print(f"Loading Florence-2 on {DEVICE.upper()} (will move to GPU during inference if on Spaces)...") # Hotfix for Florence-2 in newer transformers versions import transformers if not hasattr(transformers.PretrainedConfig, "forced_bos_token_id"): transformers.PretrainedConfig.forced_bos_token_id = None # Force _supports_sdpa to False on the actual base class import transformers.modeling_utils transformers.modeling_utils.PreTrainedModel._supports_sdpa = False self.model = AutoModelForCausalLM.from_pretrained( CONFIG.MODEL_NAME, trust_remote_code=True, torch_dtype=DTYPE, attn_implementation="eager" ).eval() # Hotfix for Florence-2 processor tokenizer compatibility if not hasattr(transformers.PreTrainedTokenizerBase, "additional_special_tokens"): transformers.PreTrainedTokenizerBase.additional_special_tokens = property( lambda self: getattr(self, "_additional_special_tokens", []) ) self.processor = AutoProcessor.from_pretrained( CONFIG.MODEL_NAME, trust_remote_code=True, ) print("Florence-2 loaded successfully") try: from paddleocr import PaddleOCR print("Loading PaddleOCR...") self.paddle_ocr = PaddleOCR(use_angle_cls=True, lang='en', show_log=False) print("PaddleOCR loaded successfully") except Exception as e: print(f"PaddleOCR load failed: {e}") self._warmup() except Exception as e: print(f"Model loading failed: {e}") raise def _warmup(self): """Run a dummy inference to warm up kernels.""" if IS_SPACES: print("Skipping warmup on ZeroGPU Spaces") return try: dummy = Image.new("RGB", (224, 224), 128) self._run_inference(dummy, "