import os import torch import base64 from PIL import Image from io import BytesIO from typing import Dict, Any from transformers import LlamaTokenizer, GenerationConfig from robohusky.model.modeling_husky_embody2 import HuskyForConditionalGeneration from decord import VideoReader, cpu import torchvision.transforms as T from torchvision.transforms.functional import InterpolationMode import tempfile DEFAULT_IMG_START_TOKEN = "" DEFAULT_IMG_END_TOKEN = "" DEFAULT_VIDEO_START_TOKEN = "" DEFAULT_VIDEO_END_TOKEN = "" class EndpointHandler: def __init__(self, model_path: str = "."): self.device = "cuda" if torch.cuda.is_available() else "cpu" self.tokenizer = LlamaTokenizer.from_pretrained(model_path, use_fast=False) self.model = HuskyForConditionalGeneration.from_pretrained( model_path, torch_dtype=torch.float16 if self.device == "cuda" else torch.float32 ).to(self.device).eval() self.gen_config = GenerationConfig( bos_token_id=1, do_sample=True, temperature=0.7, max_new_tokens=1024 ) def __call__(self, data: Dict[str, Any]) -> Dict[str, str]: inputs = self.preprocess(data) prediction = self.inference(inputs) return self.postprocess(prediction) def preprocess(self, request: Dict[str, Any]) -> Dict[str, Any]: prompt = request["inputs"] image_b64 = request.get("image", None) video_b64 = request.get("video", None) pixel_values = None if image_b64: image_bytes = base64.b64decode(image_b64) pixel_values = self._load_image(image_bytes).unsqueeze(0) # [1, 3, 224, 224] if self.device == "cuda": pixel_values = pixel_values.half() pixel_values = pixel_values.to(self.device) prompt = prompt.replace("", DEFAULT_IMG_START_TOKEN + DEFAULT_IMG_END_TOKEN) elif video_b64: video_bytes = base64.b64decode(video_b64) pixel_values = self._load_video(video_bytes) if self.device == "cuda": pixel_values = pixel_values.half() pixel_values = pixel_values.to(self.device) prompt = prompt.replace("