Spaces:
Sleeping
Sleeping
Download app.py from sosppxo/RSReasonNet: direct link, hf CLI and curl.
- Browser
- Download file 16.9 kB
-
https://huggingface.co/spaces/sosppxo/RSReasonNet/resolve/main/app.py
- Command line
-
hf download hf://spaces/sosppxo/RSReasonNet/app.py
-
curl -L -o app.py https://huggingface.co/spaces/sosppxo/RSReasonNet/resolve/main/app.py
16.9 kB
| import logging | |
| import os | |
| import re | |
| import textwrap | |
| import threading | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any, Optional, Tuple | |
| import bleach | |
| import gradio as gr | |
| import numpy as np | |
| import spaces | |
| import torch | |
| import torch.nn.functional as F | |
| from huggingface_hub import snapshot_download | |
| from PIL import Image, ImageDraw | |
| from transformers import AutoTokenizer, CLIPImageProcessor | |
| from model.LISA_sam2 import LISAForCausalLM | |
| from model.llava import conversation as conversation_lib | |
| from model.llava.mm_utils import tokenizer_image_token | |
| from model.segment_anything.utils.transforms import ResizeLongestSide | |
| from utils.utils import ( | |
| DEFAULT_IMAGE_TOKEN, | |
| DEFAULT_IM_END_TOKEN, | |
| DEFAULT_IM_START_TOKEN, | |
| IMAGE_TOKEN_INDEX, | |
| ) | |
| from ui import build_demo | |
| logging.basicConfig( | |
| level=os.getenv("LOG_LEVEL", "INFO").upper(), | |
| format="%(asctime)s %(levelname)s %(name)s: %(message)s", | |
| ) | |
| LOGGER = logging.getLogger("rsreasonnet-space") | |
| # ZeroGPU's device scheduler uses authenticated internal URLs. Suppress the | |
| # third-party HTTP client's INFO request logging so transient credentials never | |
| # appear in application logs; our own lifecycle logs remain at INFO. | |
| logging.getLogger("httpx").setLevel(logging.WARNING) | |
| VISION_MODEL_ID = "openai/clip-vit-large-patch14" | |
| VISION_MODEL_REVISION = "32bd64288804d66eefd0ccbe215aa642df71cc41" | |
| MODEL_FILES = ( | |
| "added_tokens.json", | |
| "config.json", | |
| "generation_config.json", | |
| "pytorch_model-00001-of-00002.bin", | |
| "pytorch_model-00002-of-00002.bin", | |
| "pytorch_model.bin.index.json", | |
| "special_tokens_map.json", | |
| "tokenizer.model", | |
| "tokenizer_config.json", | |
| ) | |
| VALID_PROMPT = re.compile(r"^[A-Za-z0-9\s,.!?\'\":;()\-]+$") | |
| def _read_bool(name: str, default: bool = False) -> bool: | |
| value = os.getenv(name) | |
| if value is None: | |
| return default | |
| normalized = value.strip().lower() | |
| if normalized in {"1", "true", "yes", "on"}: | |
| return True | |
| if normalized in {"0", "false", "no", "off"}: | |
| return False | |
| raise ValueError(f"{name} must be one of 1/0, true/false, yes/no, or on/off") | |
| class RuntimeConfig: | |
| model_repo_id: str | |
| model_revision: str | |
| hf_token: str | |
| precision: str | |
| load_in_4bit: bool | |
| max_new_tokens: int | |
| image_size: int = 1024 | |
| model_max_length: int = 512 | |
| conversation_type: str = "llava_v1" | |
| def from_environment(cls) -> "RuntimeConfig": | |
| precision = os.getenv("PRECISION", "fp16").strip().lower() | |
| if precision not in {"fp16", "bf16", "fp32"}: | |
| raise ValueError("PRECISION must be fp16, bf16, or fp32") | |
| try: | |
| max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", "512")) | |
| except ValueError as exc: | |
| raise ValueError("MAX_NEW_TOKENS must be an integer") from exc | |
| if not 1 <= max_new_tokens <= 512: | |
| raise ValueError("MAX_NEW_TOKENS must be between 1 and 512") | |
| return cls( | |
| model_repo_id=os.getenv( | |
| "MODEL_REPO_ID", "sosppxo/RSReasonNet-RAO-w" | |
| ).strip(), | |
| model_revision=os.getenv("MODEL_REVISION", "").strip(), | |
| hf_token=os.getenv("HF_TOKEN", "").strip(), | |
| precision=precision, | |
| load_in_4bit=_read_bool("LOAD_IN_4BIT", False), | |
| max_new_tokens=max_new_tokens, | |
| ) | |
| def validate(self) -> None: | |
| if not re.fullmatch(r"[^/\s]+/[^/\s]+", self.model_repo_id): | |
| raise ValueError("MODEL_REPO_ID must use the owner/repository form") | |
| if not re.fullmatch(r"[0-9a-fA-F]{40}", self.model_revision): | |
| raise ValueError("MODEL_REVISION must be a full 40-character commit SHA") | |
| if not self.hf_token: | |
| raise ValueError( | |
| "HF_TOKEN is missing; add a read-only token as a Space Secret" | |
| ) | |
| if self.load_in_4bit: | |
| raise ValueError( | |
| "LOAD_IN_4BIT is not supported by this ZeroGPU deployment; set it to 0" | |
| ) | |
| def _redact(message: Any, token: str = "") -> str: | |
| text = str(message) | |
| if token: | |
| text = text.replace(token, "[REDACTED]") | |
| text = re.sub(r"hf_[A-Za-z0-9]{12,}", "[REDACTED]", text) | |
| return text | |
| def _cuda_memory(stage: str) -> None: | |
| try: | |
| if not torch.cuda.is_available(): | |
| LOGGER.info("CUDA memory at %s: CUDA unavailable", stage) | |
| return | |
| gib = 1024**3 | |
| LOGGER.info( | |
| "CUDA memory at %s: allocated=%.2f GiB reserved=%.2f GiB peak=%.2f GiB", | |
| stage, | |
| torch.cuda.memory_allocated() / gib, | |
| torch.cuda.memory_reserved() / gib, | |
| torch.cuda.max_memory_allocated() / gib, | |
| ) | |
| except Exception as exc: | |
| LOGGER.info("CUDA memory at %s is not available yet: %s", stage, exc) | |
| def preprocess( | |
| image: torch.Tensor, | |
| image_size: int, | |
| pixel_mean: torch.Tensor = torch.tensor([123.675, 116.28, 103.53]).view( | |
| -1, 1, 1 | |
| ), | |
| pixel_std: torch.Tensor = torch.tensor([58.395, 57.12, 57.375]).view( | |
| -1, 1, 1 | |
| ), | |
| ) -> torch.Tensor: | |
| image = (image - pixel_mean) / pixel_std | |
| height, width = image.shape[-2:] | |
| return F.pad(image, (0, image_size - width, 0, image_size - height)) | |
| class ModelRuntime: | |
| def __init__( | |
| self, config: Optional[RuntimeConfig], configuration_error: Optional[str] | |
| ) -> None: | |
| self.config = config | |
| self.configuration_error = configuration_error | |
| self.state = "not_started" | |
| self.error = "" | |
| self.model: Any = None | |
| self.tokenizer: Any = None | |
| self.clip_image_processor: Any = None | |
| self.transform: Any = None | |
| self.torch_dtype: Any = None | |
| self.device = torch.device("cuda:0") | |
| self._lock = threading.Lock() | |
| def load(self) -> None: | |
| with self._lock: | |
| if self.state != "not_started": | |
| return | |
| self.state = "loading" | |
| self._load_worker() | |
| def snapshot(self) -> Tuple[str, str]: | |
| with self._lock: | |
| return self.state, self.error | |
| def _fail(self, exc: Exception) -> None: | |
| token = self.config.hf_token if self.config else "" | |
| message = _redact(exc, token) | |
| LOGGER.exception("Model loading failed (%s): %s", type(exc).__name__, message) | |
| with self._lock: | |
| self.error = message | |
| self.state = "failed" | |
| def _load_worker(self) -> None: | |
| try: | |
| if self.configuration_error: | |
| raise ValueError(self.configuration_error) | |
| if self.config is None: | |
| raise RuntimeError("Runtime configuration is unavailable") | |
| self.config.validate() | |
| LOGGER.info( | |
| "Downloading model repo %s at revision %s", | |
| self.config.model_repo_id, | |
| self.config.model_revision, | |
| ) | |
| model_path = snapshot_download( | |
| repo_id=self.config.model_repo_id, | |
| revision=self.config.model_revision, | |
| token=self.config.hf_token, | |
| allow_patterns=list(MODEL_FILES), | |
| max_workers=4, | |
| ) | |
| missing_files = [ | |
| filename | |
| for filename in MODEL_FILES | |
| if not (Path(model_path) / filename).is_file() | |
| ] | |
| if missing_files: | |
| raise RuntimeError( | |
| "Model snapshot is incomplete; missing: " | |
| + ", ".join(missing_files) | |
| ) | |
| LOGGER.info("Model snapshot download completed") | |
| self.torch_dtype = { | |
| "fp16": torch.float16, | |
| "bf16": torch.bfloat16, | |
| "fp32": torch.float32, | |
| }[self.config.precision] | |
| tokenizer = AutoTokenizer.from_pretrained( | |
| model_path, | |
| local_files_only=True, | |
| model_max_length=self.config.model_max_length, | |
| padding_side="right", | |
| use_fast=False, | |
| ) | |
| tokenizer.pad_token = tokenizer.unk_token | |
| seg_token_idx = tokenizer( | |
| "[SEG]", add_special_tokens=False | |
| ).input_ids[0] | |
| model_kwargs = {"torch_dtype": self.torch_dtype} | |
| LOGGER.info( | |
| "Loading model with precision=%s load_in_4bit=%s", | |
| self.config.precision, | |
| self.config.load_in_4bit, | |
| ) | |
| model = LISAForCausalLM.from_pretrained( | |
| model_path, | |
| local_files_only=True, | |
| low_cpu_mem_usage=True, | |
| sam2_config="configs/sam2.1/sam2.1_hiera_l.yaml", | |
| vision_tower=VISION_MODEL_ID, | |
| vision_tower_revision=VISION_MODEL_REVISION, | |
| seg_token_idx=seg_token_idx, | |
| **model_kwargs, | |
| ) | |
| sam2_fuser_layers = ( | |
| model.get_model().visual_model.memory_encoder.fuser.layers | |
| ) | |
| with torch.no_grad(): | |
| for layer in sam2_fuser_layers: | |
| if layer.gamma is not None: | |
| layer.gamma.fill_(1e-6) | |
| LOGGER.info( | |
| "Restored SAM2 memory fuser layer-scale defaults for %d layers", | |
| len(sam2_fuser_layers), | |
| ) | |
| model.config.eos_token_id = tokenizer.eos_token_id | |
| model.config.bos_token_id = tokenizer.bos_token_id | |
| model.config.pad_token_id = tokenizer.pad_token_id | |
| model.config.vision_tower_revision = VISION_MODEL_REVISION | |
| model.get_model().config.vision_tower_revision = VISION_MODEL_REVISION | |
| model.get_model().initialize_vision_modules(model.get_model().config) | |
| vision_tower = model.get_model().get_vision_tower() | |
| if self.config.precision == "fp16": | |
| model = model.half().cuda() | |
| elif self.config.precision == "bf16": | |
| model = model.bfloat16().cuda() | |
| else: | |
| model = model.float().cuda() | |
| vision_tower = model.get_model().get_vision_tower() | |
| vision_tower.to(device=self.device, dtype=self.torch_dtype) | |
| clip_image_processor = CLIPImageProcessor.from_pretrained( | |
| VISION_MODEL_ID, | |
| revision=VISION_MODEL_REVISION, | |
| ) | |
| transform = ResizeLongestSide(self.config.image_size) | |
| model.eval() | |
| _cuda_memory("model ready") | |
| with self._lock: | |
| self.model = model | |
| self.tokenizer = tokenizer | |
| self.clip_image_processor = clip_image_processor | |
| self.transform = transform | |
| self.state = "ready" | |
| LOGGER.info("RSReasonNet model is ready") | |
| except Exception as exc: | |
| self._fail(exc) | |
| def _message_image(message: str) -> np.ndarray: | |
| canvas = Image.new("RGB", (768, 320), color=(246, 248, 252)) | |
| draw = ImageDraw.Draw(canvas) | |
| draw.rectangle((18, 18, 750, 302), outline=(89, 105, 135), width=3) | |
| lines = textwrap.wrap(message, width=64) or [message] | |
| draw.multiline_text( | |
| (42, 54), | |
| "\n".join(lines[:9]), | |
| fill=(30, 41, 59), | |
| spacing=10, | |
| ) | |
| return np.asarray(canvas) | |
| try: | |
| RUNTIME_CONFIG = RuntimeConfig.from_environment() | |
| CONFIGURATION_ERROR = None | |
| except Exception as exc: | |
| RUNTIME_CONFIG = None | |
| CONFIGURATION_ERROR = _redact(exc) | |
| RUNTIME = ModelRuntime(RUNTIME_CONFIG, CONFIGURATION_ERROR) | |
| def inference(input_text: str, input_image: Optional[str]): | |
| if input_image is None: | |
| message = "Input image is required." | |
| return _message_image(message), f"[Error] {message}" | |
| raw_text = "" if input_text is None else str(input_text) | |
| cleaned_text = bleach.clean(raw_text, tags=[], attributes={}, strip=True).strip() | |
| if not cleaned_text: | |
| message = "Text instruction is required." | |
| return _message_image(message), f"[Error] {message}" | |
| if len(cleaned_text) > 1024 or not VALID_PROMPT.fullmatch(cleaned_text): | |
| message = ( | |
| "The instruction must be at most 1024 characters and use standard " | |
| "English letters, numbers, spaces, and punctuation." | |
| ) | |
| return _message_image(message), f"[Error] {message}" | |
| state, load_error = RUNTIME.snapshot() | |
| if state == "loading" or state == "not_started": | |
| message = "The model is still loading. Please retry in a few minutes." | |
| return _message_image(message), f"[Loading] {message}" | |
| if state != "ready": | |
| message = f"Model unavailable: {load_error or 'unknown loading error'}" | |
| return _message_image(message), f"[Error] {message}" | |
| config = RUNTIME.config | |
| if config is None: | |
| message = "Runtime configuration is unavailable." | |
| return _message_image(message), f"[Error] {message}" | |
| try: | |
| image_path = str(input_image) | |
| image_np = np.asarray(Image.open(image_path).convert("RGB")) | |
| original_size_list = [image_np.shape[:2]] | |
| conversation = conversation_lib.conv_templates[ | |
| config.conversation_type | |
| ].copy() | |
| conversation.messages = [] | |
| prompt = DEFAULT_IMAGE_TOKEN + "\n" + cleaned_text | |
| replacement = ( | |
| DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN | |
| ) | |
| prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replacement) | |
| conversation.append_message(conversation.roles[0], prompt) | |
| conversation.append_message(conversation.roles[1], "") | |
| prompt = conversation.get_prompt() | |
| with torch.inference_mode(): | |
| image_clip = RUNTIME.clip_image_processor.preprocess( | |
| image_np, return_tensors="pt" | |
| )["pixel_values"].to( | |
| device=RUNTIME.device, dtype=RUNTIME.torch_dtype | |
| ) | |
| resized_image = RUNTIME.transform.apply_image(image_np) | |
| resize_list = [resized_image.shape[:2]] | |
| image_tensor = preprocess( | |
| torch.from_numpy(resized_image) | |
| .permute(2, 0, 1) | |
| .contiguous(), | |
| config.image_size, | |
| ).unsqueeze(0) | |
| image_tensor = image_tensor.to( | |
| device=RUNTIME.device, dtype=RUNTIME.torch_dtype | |
| ) | |
| input_ids = tokenizer_image_token( | |
| prompt, RUNTIME.tokenizer, return_tensors="pt" | |
| ).unsqueeze(0) | |
| input_ids = input_ids.to(RUNTIME.device) | |
| output_ids, predicted_masks = RUNTIME.model.evaluate( | |
| [image_path], | |
| image_clip, | |
| image_tensor, | |
| input_ids, | |
| resize_list, | |
| original_size_list, | |
| max_new_tokens=config.max_new_tokens, | |
| tokenizer=RUNTIME.tokenizer, | |
| ) | |
| output_ids = output_ids[0][output_ids[0] != IMAGE_TOKEN_INDEX] | |
| text_output = RUNTIME.tokenizer.decode( | |
| output_ids, skip_special_tokens=False | |
| ) | |
| text_output = text_output.replace("\n", " ").replace(" ", " ") | |
| text_output = text_output.split("ASSISTANT:")[-1] | |
| text_output = text_output.replace("<s>", "").replace("</s>", "").strip() | |
| combined_mask = np.zeros(image_np.shape[:2], dtype=bool) | |
| for predicted_mask in predicted_masks: | |
| if predicted_mask.numel() == 0: | |
| continue | |
| mask_array = predicted_mask.detach().float().cpu().numpy() > 0 | |
| combined_mask |= np.any(mask_array, axis=0) | |
| if combined_mask.any(): | |
| output_image = image_np.copy() | |
| overlay = np.zeros_like(output_image) | |
| overlay[:, :] = np.array([255, 48, 48], dtype=np.uint8) | |
| output_image[combined_mask] = ( | |
| 0.5 * output_image[combined_mask] + 0.5 * overlay[combined_mask] | |
| ).astype(np.uint8) | |
| else: | |
| output_image = _message_image("No segmentation mask was generated.") | |
| _cuda_memory("inference complete") | |
| return output_image, f"ASSISTANT: {text_output}" | |
| except Exception as exc: | |
| token = config.hf_token | |
| message = _redact(exc, token) | |
| LOGGER.exception("Inference failed (%s): %s", type(exc).__name__, message) | |
| return _message_image("Inference failed. Check the Space logs."), ( | |
| f"[Error] Inference failed: {message}" | |
| ) | |
| DEMO = build_demo(inference) | |
| if __name__ == "__main__": | |
| # ZeroGPU's CUDA emulation optimizes model placement performed during startup. | |
| RUNTIME.load() | |
| DEMO.queue(default_concurrency_limit=1, max_size=8) | |
| DEMO.launch() | |