Update modeling_minicpmv.py

modeling_minicpmv.py

@@ -231,44 +231,48 @@ class MiniCPMV(MiniCPMVPreTrainedModel):
 
     def generate(
         self,
-        model_inputs,
+        model_inputs_batch,
         tokenizer=None,
         vision_hidden_states=None,
         stream=False,
         **kwargs
     ):
-        bs = len(model_inputs["input_ids"])
-        img_list = model_inputs["pixel_values"]
-        tgt_sizes = model_inputs["tgt_sizes"]
-        if img_list is None:
-            img_list = [[] for i in range(bs)]
-        assert bs == len(img_list)
-        if vision_hidden_states is None:
-            pixel_values = []
-            for i in range(bs):
-                img_inps = []
-                for img in img_list[i]:
-                    img_inps.append(img.to(self.device))
-                if img_inps:
-                    pixel_values.append(img_inps)
-                else:
-                    pixel_values.append([])
-            model_inputs["pixel_values"] = pixel_values
-            model_inputs['tgt_sizes'] = tgt_sizes
-        else:
-            model_inputs["vision_hidden_states"] = vision_hidden_states
-
-        (
-            input_embeds,
-            vision_hidden_states,
-        ) = self.get_vllm_embedding(model_inputs)            
+        batch = []
+        for model_inputs in model_inputs_batch:
+            bs = len(model_inputs["input_ids"])
+            img_list = model_inputs["pixel_values"]
+            tgt_sizes = model_inputs["tgt_sizes"]
+            if img_list is None:
+                img_list = [[] for i in range(bs)]
+            assert bs == len(img_list)
+            if vision_hidden_states is None:
+                pixel_values = []
+                for i in range(bs):
+                    img_inps = []
+                    for img in img_list[i]:
+                        img_inps.append(img.to(self.device))
+                    if img_inps:
+                        pixel_values.append(img_inps)
+                    else:
+                        pixel_values.append([])
+                model_inputs["pixel_values"] = pixel_values
+                model_inputs['tgt_sizes'] = tgt_sizes
+            else:
+                model_inputs["vision_hidden_states"] = vision_hidden_states
+    
+            (
+                input_embeds,
+                vision_hidden_states,
+            ) = self.get_vllm_embedding(model_inputs)  
+            batch.append(input_embeds)
+            
 
         # output_ids = self._decode(input_embeds, tokenizer, **kwargs)
         if stream:
             kwargs.pop("decode_text")
-            result = self._decode_stream(input_embeds, tokenizer, **kwargs)
+            result = self._decode_stream(batch, tokenizer, **kwargs)
         else:
-            result = self._decode(input_embeds, tokenizer, **kwargs)
+            result = self._decode(batch, tokenizer, **kwargs)
 
         return result
 
@@ -366,5 +370,5 @@ class MiniCPMV(MiniCPMVPreTrainedModel):
             return stream_gen()
 
         else:
-            answer = res[0]
+            answer = res
             return answer