import logging import os import re import textwrap import threading from dataclasses import dataclass from pathlib import Path from typing import Any, Optional, Tuple import bleach import gradio as gr import numpy as np import spaces import torch import torch.nn.functional as F from huggingface_hub import snapshot_download from PIL import Image, ImageDraw from transformers import AutoTokenizer, CLIPImageProcessor from model.LISA_sam2 import LISAForCausalLM from model.llava import conversation as conversation_lib from model.llava.mm_utils import tokenizer_image_token from model.segment_anything.utils.transforms import ResizeLongestSide from utils.utils import ( DEFAULT_IMAGE_TOKEN, DEFAULT_IM_END_TOKEN, DEFAULT_IM_START_TOKEN, IMAGE_TOKEN_INDEX, ) from ui import build_demo logging.basicConfig( level=os.getenv("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(name)s: %(message)s", ) LOGGER = logging.getLogger("rsreasonnet-space") # ZeroGPU's device scheduler uses authenticated internal URLs. Suppress the # third-party HTTP client's INFO request logging so transient credentials never # appear in application logs; our own lifecycle logs remain at INFO. logging.getLogger("httpx").setLevel(logging.WARNING) VISION_MODEL_ID = "openai/clip-vit-large-patch14" VISION_MODEL_REVISION = "32bd64288804d66eefd0ccbe215aa642df71cc41" MODEL_FILES = ( "added_tokens.json", "config.json", "generation_config.json", "pytorch_model-00001-of-00002.bin", "pytorch_model-00002-of-00002.bin", "pytorch_model.bin.index.json", "special_tokens_map.json", "tokenizer.model", "tokenizer_config.json", ) VALID_PROMPT = re.compile(r"^[A-Za-z0-9\s,.!?\'\":;()\-]+$") def _read_bool(name: str, default: bool = False) -> bool: value = os.getenv(name) if value is None: return default normalized = value.strip().lower() if normalized in {"1", "true", "yes", "on"}: return True if normalized in {"0", "false", "no", "off"}: return False raise ValueError(f"{name} must be one of 1/0, true/false, yes/no, or on/off") @dataclass(frozen=True) class RuntimeConfig: model_repo_id: str model_revision: str hf_token: str precision: str load_in_4bit: bool max_new_tokens: int image_size: int = 1024 model_max_length: int = 512 conversation_type: str = "llava_v1" @classmethod def from_environment(cls) -> "RuntimeConfig": precision = os.getenv("PRECISION", "fp16").strip().lower() if precision not in {"fp16", "bf16", "fp32"}: raise ValueError("PRECISION must be fp16, bf16, or fp32") try: max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", "512")) except ValueError as exc: raise ValueError("MAX_NEW_TOKENS must be an integer") from exc if not 1 <= max_new_tokens <= 512: raise ValueError("MAX_NEW_TOKENS must be between 1 and 512") return cls( model_repo_id=os.getenv( "MODEL_REPO_ID", "sosppxo/RSReasonNet-RAO-w" ).strip(), model_revision=os.getenv("MODEL_REVISION", "").strip(), hf_token=os.getenv("HF_TOKEN", "").strip(), precision=precision, load_in_4bit=_read_bool("LOAD_IN_4BIT", False), max_new_tokens=max_new_tokens, ) def validate(self) -> None: if not re.fullmatch(r"[^/\s]+/[^/\s]+", self.model_repo_id): raise ValueError("MODEL_REPO_ID must use the owner/repository form") if not re.fullmatch(r"[0-9a-fA-F]{40}", self.model_revision): raise ValueError("MODEL_REVISION must be a full 40-character commit SHA") if not self.hf_token: raise ValueError( "HF_TOKEN is missing; add a read-only token as a Space Secret" ) if self.load_in_4bit: raise ValueError( "LOAD_IN_4BIT is not supported by this ZeroGPU deployment; set it to 0" ) def _redact(message: Any, token: str = "") -> str: text = str(message) if token: text = text.replace(token, "[REDACTED]") text = re.sub(r"hf_[A-Za-z0-9]{12,}", "[REDACTED]", text) return text def _cuda_memory(stage: str) -> None: try: if not torch.cuda.is_available(): LOGGER.info("CUDA memory at %s: CUDA unavailable", stage) return gib = 1024**3 LOGGER.info( "CUDA memory at %s: allocated=%.2f GiB reserved=%.2f GiB peak=%.2f GiB", stage, torch.cuda.memory_allocated() / gib, torch.cuda.memory_reserved() / gib, torch.cuda.max_memory_allocated() / gib, ) except Exception as exc: LOGGER.info("CUDA memory at %s is not available yet: %s", stage, exc) def preprocess( image: torch.Tensor, image_size: int, pixel_mean: torch.Tensor = torch.tensor([123.675, 116.28, 103.53]).view( -1, 1, 1 ), pixel_std: torch.Tensor = torch.tensor([58.395, 57.12, 57.375]).view( -1, 1, 1 ), ) -> torch.Tensor: image = (image - pixel_mean) / pixel_std height, width = image.shape[-2:] return F.pad(image, (0, image_size - width, 0, image_size - height)) class ModelRuntime: def __init__( self, config: Optional[RuntimeConfig], configuration_error: Optional[str] ) -> None: self.config = config self.configuration_error = configuration_error self.state = "not_started" self.error = "" self.model: Any = None self.tokenizer: Any = None self.clip_image_processor: Any = None self.transform: Any = None self.torch_dtype: Any = None self.device = torch.device("cuda:0") self._lock = threading.Lock() def load(self) -> None: with self._lock: if self.state != "not_started": return self.state = "loading" self._load_worker() def snapshot(self) -> Tuple[str, str]: with self._lock: return self.state, self.error def _fail(self, exc: Exception) -> None: token = self.config.hf_token if self.config else "" message = _redact(exc, token) LOGGER.exception("Model loading failed (%s): %s", type(exc).__name__, message) with self._lock: self.error = message self.state = "failed" def _load_worker(self) -> None: try: if self.configuration_error: raise ValueError(self.configuration_error) if self.config is None: raise RuntimeError("Runtime configuration is unavailable") self.config.validate() LOGGER.info( "Downloading model repo %s at revision %s", self.config.model_repo_id, self.config.model_revision, ) model_path = snapshot_download( repo_id=self.config.model_repo_id, revision=self.config.model_revision, token=self.config.hf_token, allow_patterns=list(MODEL_FILES), max_workers=4, ) missing_files = [ filename for filename in MODEL_FILES if not (Path(model_path) / filename).is_file() ] if missing_files: raise RuntimeError( "Model snapshot is incomplete; missing: " + ", ".join(missing_files) ) LOGGER.info("Model snapshot download completed") self.torch_dtype = { "fp16": torch.float16, "bf16": torch.bfloat16, "fp32": torch.float32, }[self.config.precision] tokenizer = AutoTokenizer.from_pretrained( model_path, local_files_only=True, model_max_length=self.config.model_max_length, padding_side="right", use_fast=False, ) tokenizer.pad_token = tokenizer.unk_token seg_token_idx = tokenizer( "[SEG]", add_special_tokens=False ).input_ids[0] model_kwargs = {"torch_dtype": self.torch_dtype} LOGGER.info( "Loading model with precision=%s load_in_4bit=%s", self.config.precision, self.config.load_in_4bit, ) model = LISAForCausalLM.from_pretrained( model_path, local_files_only=True, low_cpu_mem_usage=True, sam2_config="configs/sam2.1/sam2.1_hiera_l.yaml", vision_tower=VISION_MODEL_ID, vision_tower_revision=VISION_MODEL_REVISION, seg_token_idx=seg_token_idx, **model_kwargs, ) sam2_fuser_layers = ( model.get_model().visual_model.memory_encoder.fuser.layers ) with torch.no_grad(): for layer in sam2_fuser_layers: if layer.gamma is not None: layer.gamma.fill_(1e-6) LOGGER.info( "Restored SAM2 memory fuser layer-scale defaults for %d layers", len(sam2_fuser_layers), ) model.config.eos_token_id = tokenizer.eos_token_id model.config.bos_token_id = tokenizer.bos_token_id model.config.pad_token_id = tokenizer.pad_token_id model.config.vision_tower_revision = VISION_MODEL_REVISION model.get_model().config.vision_tower_revision = VISION_MODEL_REVISION model.get_model().initialize_vision_modules(model.get_model().config) vision_tower = model.get_model().get_vision_tower() if self.config.precision == "fp16": model = model.half().cuda() elif self.config.precision == "bf16": model = model.bfloat16().cuda() else: model = model.float().cuda() vision_tower = model.get_model().get_vision_tower() vision_tower.to(device=self.device, dtype=self.torch_dtype) clip_image_processor = CLIPImageProcessor.from_pretrained( VISION_MODEL_ID, revision=VISION_MODEL_REVISION, ) transform = ResizeLongestSide(self.config.image_size) model.eval() _cuda_memory("model ready") with self._lock: self.model = model self.tokenizer = tokenizer self.clip_image_processor = clip_image_processor self.transform = transform self.state = "ready" LOGGER.info("RSReasonNet model is ready") except Exception as exc: self._fail(exc) def _message_image(message: str) -> np.ndarray: canvas = Image.new("RGB", (768, 320), color=(246, 248, 252)) draw = ImageDraw.Draw(canvas) draw.rectangle((18, 18, 750, 302), outline=(89, 105, 135), width=3) lines = textwrap.wrap(message, width=64) or [message] draw.multiline_text( (42, 54), "\n".join(lines[:9]), fill=(30, 41, 59), spacing=10, ) return np.asarray(canvas) try: RUNTIME_CONFIG = RuntimeConfig.from_environment() CONFIGURATION_ERROR = None except Exception as exc: RUNTIME_CONFIG = None CONFIGURATION_ERROR = _redact(exc) RUNTIME = ModelRuntime(RUNTIME_CONFIG, CONFIGURATION_ERROR) @spaces.GPU(size="large", duration=300) def inference(input_text: str, input_image: Optional[str]): if input_image is None: message = "Input image is required." return _message_image(message), f"[Error] {message}" raw_text = "" if input_text is None else str(input_text) cleaned_text = bleach.clean(raw_text, tags=[], attributes={}, strip=True).strip() if not cleaned_text: message = "Text instruction is required." return _message_image(message), f"[Error] {message}" if len(cleaned_text) > 1024 or not VALID_PROMPT.fullmatch(cleaned_text): message = ( "The instruction must be at most 1024 characters and use standard " "English letters, numbers, spaces, and punctuation." ) return _message_image(message), f"[Error] {message}" state, load_error = RUNTIME.snapshot() if state == "loading" or state == "not_started": message = "The model is still loading. Please retry in a few minutes." return _message_image(message), f"[Loading] {message}" if state != "ready": message = f"Model unavailable: {load_error or 'unknown loading error'}" return _message_image(message), f"[Error] {message}" config = RUNTIME.config if config is None: message = "Runtime configuration is unavailable." return _message_image(message), f"[Error] {message}" try: image_path = str(input_image) image_np = np.asarray(Image.open(image_path).convert("RGB")) original_size_list = [image_np.shape[:2]] conversation = conversation_lib.conv_templates[ config.conversation_type ].copy() conversation.messages = [] prompt = DEFAULT_IMAGE_TOKEN + "\n" + cleaned_text replacement = ( DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN ) prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replacement) conversation.append_message(conversation.roles[0], prompt) conversation.append_message(conversation.roles[1], "") prompt = conversation.get_prompt() with torch.inference_mode(): image_clip = RUNTIME.clip_image_processor.preprocess( image_np, return_tensors="pt" )["pixel_values"].to( device=RUNTIME.device, dtype=RUNTIME.torch_dtype ) resized_image = RUNTIME.transform.apply_image(image_np) resize_list = [resized_image.shape[:2]] image_tensor = preprocess( torch.from_numpy(resized_image) .permute(2, 0, 1) .contiguous(), config.image_size, ).unsqueeze(0) image_tensor = image_tensor.to( device=RUNTIME.device, dtype=RUNTIME.torch_dtype ) input_ids = tokenizer_image_token( prompt, RUNTIME.tokenizer, return_tensors="pt" ).unsqueeze(0) input_ids = input_ids.to(RUNTIME.device) output_ids, predicted_masks = RUNTIME.model.evaluate( [image_path], image_clip, image_tensor, input_ids, resize_list, original_size_list, max_new_tokens=config.max_new_tokens, tokenizer=RUNTIME.tokenizer, ) output_ids = output_ids[0][output_ids[0] != IMAGE_TOKEN_INDEX] text_output = RUNTIME.tokenizer.decode( output_ids, skip_special_tokens=False ) text_output = text_output.replace("\n", " ").replace(" ", " ") text_output = text_output.split("ASSISTANT:")[-1] text_output = text_output.replace("", "").replace("", "").strip() combined_mask = np.zeros(image_np.shape[:2], dtype=bool) for predicted_mask in predicted_masks: if predicted_mask.numel() == 0: continue mask_array = predicted_mask.detach().float().cpu().numpy() > 0 combined_mask |= np.any(mask_array, axis=0) if combined_mask.any(): output_image = image_np.copy() overlay = np.zeros_like(output_image) overlay[:, :] = np.array([255, 48, 48], dtype=np.uint8) output_image[combined_mask] = ( 0.5 * output_image[combined_mask] + 0.5 * overlay[combined_mask] ).astype(np.uint8) else: output_image = _message_image("No segmentation mask was generated.") _cuda_memory("inference complete") return output_image, f"ASSISTANT: {text_output}" except Exception as exc: token = config.hf_token message = _redact(exc, token) LOGGER.exception("Inference failed (%s): %s", type(exc).__name__, message) return _message_image("Inference failed. Check the Space logs."), ( f"[Error] Inference failed: {message}" ) DEMO = build_demo(inference) if __name__ == "__main__": # ZeroGPU's CUDA emulation optimizes model placement performed during startup. RUNTIME.load() DEMO.queue(default_concurrency_limit=1, max_size=8) DEMO.launch()