import math import re from transformers.feature_extraction_utils import BatchFeature from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack class LocateAnythingProcessorKwargs(ProcessingKwargs, total=False): _defaults = {} class LocateAnythingProcessor(ProcessorMixin): attributes = ["image_processor", "tokenizer"] valid_kwargs = ["chat_template", "image_token", "image_start_token", "image_end_token"] tokenizer_class = "AutoTokenizer" def __init__( self, image_processor=None, tokenizer=None, chat_template=None, image_token="", image_start_token="", image_end_token="", **kwargs, ): self.image_token = image_token self.image_start_token = image_start_token self.image_end_token = image_end_token self.image_token_id = tokenizer.convert_tokens_to_ids(image_token) super().__init__(image_processor, tokenizer, chat_template=chat_template) def __call__(self, images=None, text=None, **kwargs: Unpack[LocateAnythingProcessorKwargs]): output_kwargs = self._merge_kwargs( LocateAnythingProcessorKwargs, tokenizer_init_kwargs=self.tokenizer.init_kwargs, **kwargs, ) text = [text] if isinstance(text, str) else list(text or []) image_inputs = {} if images is not None: image_inputs = self.image_processor(images, **output_kwargs["images_kwargs"]) grids = image_inputs["image_grid_thw"] merge_area = math.prod(self.image_processor.merge_kernel_size) image_index = 0 def replace_image(match): nonlocal image_index if image_index >= len(grids): raise ValueError("Prompt contains more image placeholders than supplied images") number = match.group(1) _, height, width = grids[image_index].tolist() image_index += 1 token_count = height * width // merge_area return ( f"{self.image_start_token}" f"{self.image_token * token_count}{self.image_end_token}" ) text = [re.sub(r"", replace_image, sample) for sample in text] if image_index != len(grids): raise ValueError("Supplied image count does not match image placeholders in the prompt") return_tensors = output_kwargs["text_kwargs"].pop("return_tensors", None) text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"]) return BatchFeature(data={**text_inputs, **image_inputs}, tensor_type=return_tensors) def batch_decode(self, *args, **kwargs): return self.tokenizer.batch_decode(*args, **kwargs) def decode(self, *args, **kwargs): return self.tokenizer.decode(*args, **kwargs) @property def model_input_names(self): names = self.tokenizer.model_input_names + self.image_processor.model_input_names return list(dict.fromkeys(names)) __all__ = ["LocateAnythingProcessor"]