RSReasonNet / app.py
sosppxo's picture
Refresh demo layout and add three annotated remote-sensing examples
a13210a verified
Raw History Blame Contribute Delete
16.9 kB
import logging
import os
import re
import textwrap
import threading
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Optional, Tuple
import bleach
import gradio as gr
import numpy as np
import spaces
import torch
import torch.nn.functional as F
from huggingface_hub import snapshot_download
from PIL import Image, ImageDraw
from transformers import AutoTokenizer, CLIPImageProcessor
from model.LISA_sam2 import LISAForCausalLM
from model.llava import conversation as conversation_lib
from model.llava.mm_utils import tokenizer_image_token
from model.segment_anything.utils.transforms import ResizeLongestSide
from utils.utils import (
DEFAULT_IMAGE_TOKEN,
DEFAULT_IM_END_TOKEN,
DEFAULT_IM_START_TOKEN,
IMAGE_TOKEN_INDEX,
)
from ui import build_demo
logging.basicConfig(
level=os.getenv("LOG_LEVEL", "INFO").upper(),
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
LOGGER = logging.getLogger("rsreasonnet-space")
# ZeroGPU's device scheduler uses authenticated internal URLs. Suppress the
# third-party HTTP client's INFO request logging so transient credentials never
# appear in application logs; our own lifecycle logs remain at INFO.
logging.getLogger("httpx").setLevel(logging.WARNING)
VISION_MODEL_ID = "openai/clip-vit-large-patch14"
VISION_MODEL_REVISION = "32bd64288804d66eefd0ccbe215aa642df71cc41"
MODEL_FILES = (
"added_tokens.json",
"config.json",
"generation_config.json",
"pytorch_model-00001-of-00002.bin",
"pytorch_model-00002-of-00002.bin",
"pytorch_model.bin.index.json",
"special_tokens_map.json",
"tokenizer.model",
"tokenizer_config.json",
)
VALID_PROMPT = re.compile(r"^[A-Za-z0-9\s,.!?\'\":;()\-]+$")
def _read_bool(name: str, default: bool = False) -> bool:
value = os.getenv(name)
if value is None:
return default
normalized = value.strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
raise ValueError(f"{name} must be one of 1/0, true/false, yes/no, or on/off")
@dataclass(frozen=True)
class RuntimeConfig:
model_repo_id: str
model_revision: str
hf_token: str
precision: str
load_in_4bit: bool
max_new_tokens: int
image_size: int = 1024
model_max_length: int = 512
conversation_type: str = "llava_v1"
@classmethod
def from_environment(cls) -> "RuntimeConfig":
precision = os.getenv("PRECISION", "fp16").strip().lower()
if precision not in {"fp16", "bf16", "fp32"}:
raise ValueError("PRECISION must be fp16, bf16, or fp32")
try:
max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", "512"))
except ValueError as exc:
raise ValueError("MAX_NEW_TOKENS must be an integer") from exc
if not 1 <= max_new_tokens <= 512:
raise ValueError("MAX_NEW_TOKENS must be between 1 and 512")
return cls(
model_repo_id=os.getenv(
"MODEL_REPO_ID", "sosppxo/RSReasonNet-RAO-w"
).strip(),
model_revision=os.getenv("MODEL_REVISION", "").strip(),
hf_token=os.getenv("HF_TOKEN", "").strip(),
precision=precision,
load_in_4bit=_read_bool("LOAD_IN_4BIT", False),
max_new_tokens=max_new_tokens,
)
def validate(self) -> None:
if not re.fullmatch(r"[^/\s]+/[^/\s]+", self.model_repo_id):
raise ValueError("MODEL_REPO_ID must use the owner/repository form")
if not re.fullmatch(r"[0-9a-fA-F]{40}", self.model_revision):
raise ValueError("MODEL_REVISION must be a full 40-character commit SHA")
if not self.hf_token:
raise ValueError(
"HF_TOKEN is missing; add a read-only token as a Space Secret"
)
if self.load_in_4bit:
raise ValueError(
"LOAD_IN_4BIT is not supported by this ZeroGPU deployment; set it to 0"
)
def _redact(message: Any, token: str = "") -> str:
text = str(message)
if token:
text = text.replace(token, "[REDACTED]")
text = re.sub(r"hf_[A-Za-z0-9]{12,}", "[REDACTED]", text)
return text
def _cuda_memory(stage: str) -> None:
try:
if not torch.cuda.is_available():
LOGGER.info("CUDA memory at %s: CUDA unavailable", stage)
return
gib = 1024**3
LOGGER.info(
"CUDA memory at %s: allocated=%.2f GiB reserved=%.2f GiB peak=%.2f GiB",
stage,
torch.cuda.memory_allocated() / gib,
torch.cuda.memory_reserved() / gib,
torch.cuda.max_memory_allocated() / gib,
)
except Exception as exc:
LOGGER.info("CUDA memory at %s is not available yet: %s", stage, exc)
def preprocess(
image: torch.Tensor,
image_size: int,
pixel_mean: torch.Tensor = torch.tensor([123.675, 116.28, 103.53]).view(
-1, 1, 1
),
pixel_std: torch.Tensor = torch.tensor([58.395, 57.12, 57.375]).view(
-1, 1, 1
),
) -> torch.Tensor:
image = (image - pixel_mean) / pixel_std
height, width = image.shape[-2:]
return F.pad(image, (0, image_size - width, 0, image_size - height))
class ModelRuntime:
def __init__(
self, config: Optional[RuntimeConfig], configuration_error: Optional[str]
) -> None:
self.config = config
self.configuration_error = configuration_error
self.state = "not_started"
self.error = ""
self.model: Any = None
self.tokenizer: Any = None
self.clip_image_processor: Any = None
self.transform: Any = None
self.torch_dtype: Any = None
self.device = torch.device("cuda:0")
self._lock = threading.Lock()
def load(self) -> None:
with self._lock:
if self.state != "not_started":
return
self.state = "loading"
self._load_worker()
def snapshot(self) -> Tuple[str, str]:
with self._lock:
return self.state, self.error
def _fail(self, exc: Exception) -> None:
token = self.config.hf_token if self.config else ""
message = _redact(exc, token)
LOGGER.exception("Model loading failed (%s): %s", type(exc).__name__, message)
with self._lock:
self.error = message
self.state = "failed"
def _load_worker(self) -> None:
try:
if self.configuration_error:
raise ValueError(self.configuration_error)
if self.config is None:
raise RuntimeError("Runtime configuration is unavailable")
self.config.validate()
LOGGER.info(
"Downloading model repo %s at revision %s",
self.config.model_repo_id,
self.config.model_revision,
)
model_path = snapshot_download(
repo_id=self.config.model_repo_id,
revision=self.config.model_revision,
token=self.config.hf_token,
allow_patterns=list(MODEL_FILES),
max_workers=4,
)
missing_files = [
filename
for filename in MODEL_FILES
if not (Path(model_path) / filename).is_file()
]
if missing_files:
raise RuntimeError(
"Model snapshot is incomplete; missing: "
+ ", ".join(missing_files)
)
LOGGER.info("Model snapshot download completed")
self.torch_dtype = {
"fp16": torch.float16,
"bf16": torch.bfloat16,
"fp32": torch.float32,
}[self.config.precision]
tokenizer = AutoTokenizer.from_pretrained(
model_path,
local_files_only=True,
model_max_length=self.config.model_max_length,
padding_side="right",
use_fast=False,
)
tokenizer.pad_token = tokenizer.unk_token
seg_token_idx = tokenizer(
"[SEG]", add_special_tokens=False
).input_ids[0]
model_kwargs = {"torch_dtype": self.torch_dtype}
LOGGER.info(
"Loading model with precision=%s load_in_4bit=%s",
self.config.precision,
self.config.load_in_4bit,
)
model = LISAForCausalLM.from_pretrained(
model_path,
local_files_only=True,
low_cpu_mem_usage=True,
sam2_config="configs/sam2.1/sam2.1_hiera_l.yaml",
vision_tower=VISION_MODEL_ID,
vision_tower_revision=VISION_MODEL_REVISION,
seg_token_idx=seg_token_idx,
**model_kwargs,
)
sam2_fuser_layers = (
model.get_model().visual_model.memory_encoder.fuser.layers
)
with torch.no_grad():
for layer in sam2_fuser_layers:
if layer.gamma is not None:
layer.gamma.fill_(1e-6)
LOGGER.info(
"Restored SAM2 memory fuser layer-scale defaults for %d layers",
len(sam2_fuser_layers),
)
model.config.eos_token_id = tokenizer.eos_token_id
model.config.bos_token_id = tokenizer.bos_token_id
model.config.pad_token_id = tokenizer.pad_token_id
model.config.vision_tower_revision = VISION_MODEL_REVISION
model.get_model().config.vision_tower_revision = VISION_MODEL_REVISION
model.get_model().initialize_vision_modules(model.get_model().config)
vision_tower = model.get_model().get_vision_tower()
if self.config.precision == "fp16":
model = model.half().cuda()
elif self.config.precision == "bf16":
model = model.bfloat16().cuda()
else:
model = model.float().cuda()
vision_tower = model.get_model().get_vision_tower()
vision_tower.to(device=self.device, dtype=self.torch_dtype)
clip_image_processor = CLIPImageProcessor.from_pretrained(
VISION_MODEL_ID,
revision=VISION_MODEL_REVISION,
)
transform = ResizeLongestSide(self.config.image_size)
model.eval()
_cuda_memory("model ready")
with self._lock:
self.model = model
self.tokenizer = tokenizer
self.clip_image_processor = clip_image_processor
self.transform = transform
self.state = "ready"
LOGGER.info("RSReasonNet model is ready")
except Exception as exc:
self._fail(exc)
def _message_image(message: str) -> np.ndarray:
canvas = Image.new("RGB", (768, 320), color=(246, 248, 252))
draw = ImageDraw.Draw(canvas)
draw.rectangle((18, 18, 750, 302), outline=(89, 105, 135), width=3)
lines = textwrap.wrap(message, width=64) or [message]
draw.multiline_text(
(42, 54),
"\n".join(lines[:9]),
fill=(30, 41, 59),
spacing=10,
)
return np.asarray(canvas)
try:
RUNTIME_CONFIG = RuntimeConfig.from_environment()
CONFIGURATION_ERROR = None
except Exception as exc:
RUNTIME_CONFIG = None
CONFIGURATION_ERROR = _redact(exc)
RUNTIME = ModelRuntime(RUNTIME_CONFIG, CONFIGURATION_ERROR)
@spaces.GPU(size="large", duration=300)
def inference(input_text: str, input_image: Optional[str]):
if input_image is None:
message = "Input image is required."
return _message_image(message), f"[Error] {message}"
raw_text = "" if input_text is None else str(input_text)
cleaned_text = bleach.clean(raw_text, tags=[], attributes={}, strip=True).strip()
if not cleaned_text:
message = "Text instruction is required."
return _message_image(message), f"[Error] {message}"
if len(cleaned_text) > 1024 or not VALID_PROMPT.fullmatch(cleaned_text):
message = (
"The instruction must be at most 1024 characters and use standard "
"English letters, numbers, spaces, and punctuation."
)
return _message_image(message), f"[Error] {message}"
state, load_error = RUNTIME.snapshot()
if state == "loading" or state == "not_started":
message = "The model is still loading. Please retry in a few minutes."
return _message_image(message), f"[Loading] {message}"
if state != "ready":
message = f"Model unavailable: {load_error or 'unknown loading error'}"
return _message_image(message), f"[Error] {message}"
config = RUNTIME.config
if config is None:
message = "Runtime configuration is unavailable."
return _message_image(message), f"[Error] {message}"
try:
image_path = str(input_image)
image_np = np.asarray(Image.open(image_path).convert("RGB"))
original_size_list = [image_np.shape[:2]]
conversation = conversation_lib.conv_templates[
config.conversation_type
].copy()
conversation.messages = []
prompt = DEFAULT_IMAGE_TOKEN + "\n" + cleaned_text
replacement = (
DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN
)
prompt = prompt.replace(DEFAULT_IMAGE_TOKEN, replacement)
conversation.append_message(conversation.roles[0], prompt)
conversation.append_message(conversation.roles[1], "")
prompt = conversation.get_prompt()
with torch.inference_mode():
image_clip = RUNTIME.clip_image_processor.preprocess(
image_np, return_tensors="pt"
)["pixel_values"].to(
device=RUNTIME.device, dtype=RUNTIME.torch_dtype
)
resized_image = RUNTIME.transform.apply_image(image_np)
resize_list = [resized_image.shape[:2]]
image_tensor = preprocess(
torch.from_numpy(resized_image)
.permute(2, 0, 1)
.contiguous(),
config.image_size,
).unsqueeze(0)
image_tensor = image_tensor.to(
device=RUNTIME.device, dtype=RUNTIME.torch_dtype
)
input_ids = tokenizer_image_token(
prompt, RUNTIME.tokenizer, return_tensors="pt"
).unsqueeze(0)
input_ids = input_ids.to(RUNTIME.device)
output_ids, predicted_masks = RUNTIME.model.evaluate(
[image_path],
image_clip,
image_tensor,
input_ids,
resize_list,
original_size_list,
max_new_tokens=config.max_new_tokens,
tokenizer=RUNTIME.tokenizer,
)
output_ids = output_ids[0][output_ids[0] != IMAGE_TOKEN_INDEX]
text_output = RUNTIME.tokenizer.decode(
output_ids, skip_special_tokens=False
)
text_output = text_output.replace("\n", " ").replace(" ", " ")
text_output = text_output.split("ASSISTANT:")[-1]
text_output = text_output.replace("<s>", "").replace("</s>", "").strip()
combined_mask = np.zeros(image_np.shape[:2], dtype=bool)
for predicted_mask in predicted_masks:
if predicted_mask.numel() == 0:
continue
mask_array = predicted_mask.detach().float().cpu().numpy() > 0
combined_mask |= np.any(mask_array, axis=0)
if combined_mask.any():
output_image = image_np.copy()
overlay = np.zeros_like(output_image)
overlay[:, :] = np.array([255, 48, 48], dtype=np.uint8)
output_image[combined_mask] = (
0.5 * output_image[combined_mask] + 0.5 * overlay[combined_mask]
).astype(np.uint8)
else:
output_image = _message_image("No segmentation mask was generated.")
_cuda_memory("inference complete")
return output_image, f"ASSISTANT: {text_output}"
except Exception as exc:
token = config.hf_token
message = _redact(exc, token)
LOGGER.exception("Inference failed (%s): %s", type(exc).__name__, message)
return _message_image("Inference failed. Check the Space logs."), (
f"[Error] Inference failed: {message}"
)
DEMO = build_demo(inference)
if __name__ == "__main__":
# ZeroGPU's CUDA emulation optimizes model placement performed during startup.
RUNTIME.load()
DEMO.queue(default_concurrency_limit=1, max_size=8)
DEMO.launch()