fix-tokenizer

processing_phi3_v.py

@@ -150,7 +150,7 @@ class Phi3VProcessor(ProcessorMixin):
             return BatchFeature(data={**model_inputs})
 
         pattern = r"<\|image_\d+\|>"
-        prompt_chunks = [self.tokenizer(chunk).input_ids for chunk in re.split(pattern, texts)] 
+        prompt_chunks = [self.tokenizer(chunk, return_tensors=return_tensors, padding=padding, truncation=truncation, max_length=max_length).input_ids for chunk in re.split(pattern, texts)] 
 
         if 'num_img_tokens' in images:
             num_img_tokens = images['num_img_tokens']