Upload 2 files

model_index.json

@@ -6,7 +6,7 @@
     "CLIPImageProcessor"
   ],
   "multimodal_encoder": [
-    "transformers_modules.modeling_emu",
+    "transformers_modules.multimodal_encoder.modeling_emu",
     "EmuForCausalLM"
   ],
   "safety_checker": [

pipeline_emu2_gen.py

@@ -8,14 +8,14 @@
 #    Email         : zhangfan@baai.ac.cn
 #    Institute     : Beijing Academy of Artificial Intelligence (BAAI)
 #    Create On     : 2023-12-19 10:45
-#    Last Modified : 2023-12-19 14:01
-#    File Name     : pipeline.py
+#    Last Modified : 2023-12-25 07:59
+#    File Name     : pipeline_emu2_gen.py
 #    Description   :
 #
 # ===========================================================================================
 
 from dataclasses import dataclass
-from typing import List, Optional, Union
+from typing import List, Optional
 
 from PIL import Image
 import numpy as np
@@ -38,8 +38,8 @@ DEFAULT_IMG_PLACEHOLDER = "[<IMG_PLH>]"
 
 @dataclass
 class EmuVisualGenerationPipelineOutput(BaseOutput):
-    images: Union[List[Image.Image], np.ndarray]
-    nsfw_content_detected: Optional[List[bool]]
+    image: Image.Image
+    nsfw_content_detected: Optional[bool]
 
 
 class EmuVisualGenerationPipeline(DiffusionPipeline):
@@ -76,7 +76,7 @@ class EmuVisualGenerationPipeline(DiffusionPipeline):
             TF.Normalize(mean=eva_mean, std=eva_std),
         ])
 
-        self.negative_prompt = None
+        self.negative_prompt = {}
 
     def device(self, module):
         return next(module.parameters()).device
@@ -166,7 +166,10 @@ class EmuVisualGenerationPipeline(DiffusionPipeline):
 
         # 7. Convert to PIL
         images = self.numpy_to_pil(images)
-        return EmuVisualGenerationPipelineOutput(images=images, nsfw_content_detected=has_nsfw_concept)
+        return EmuVisualGenerationPipelineOutput(
+            image=images[0],
+            nsfw_content_detected=None if has_nsfw_concept is None else has_nsfw_concept[0],
+        )
 
     def _prepare_and_encode_inputs(
         self,
@@ -177,11 +180,14 @@ class EmuVisualGenerationPipeline(DiffusionPipeline):
         device = self.device(self.multimodal_encoder.model.visual)
         dtype = self.dtype(self.multimodal_encoder.model.visual)
 
+        has_image, has_text = False, False
         text_prompt, image_prompt = "", []
         for x in inputs:
             if isinstance(x, str):
+                has_text = True
                 text_prompt += x
             else:
+                has_image = True
                 text_prompt += placeholder
                 image_prompt.append(self.transform(x))
 
@@ -191,11 +197,21 @@ class EmuVisualGenerationPipeline(DiffusionPipeline):
             image_prompt = torch.stack(image_prompt)
             image_prompt = image_prompt.type(dtype).to(device)
 
-        prompt = self.multimodal_encoder.generate_image(text=[text_prompt], image=image_prompt, tokenizer=self.tokenizer)
-        if do_classifier_free_guidance:
-            if self.negative_prompt is None:
-                self.negative_prompt = self.multimodal_encoder.generate_image(text=[""], tokenizer=self.tokenizer)
-            prompt = torch.cat([prompt, self.negative_prompt], dim=0)
+        if has_image and not has_text:
+            prompt = self.multimodal_encoder.model.encode_image(image=image_prompt)
+            if do_classifier_free_guidance:
+                key = "[NULL_IMAGE]"
+                if key not in self.negative_prompt:
+                    negative_image = torch.zeros_like(image_prompt)
+                    self.negative_prompt[key] = self.multimodal_encoder.model.encode_image(image=negative_image)
+                prompt = torch.cat([prompt, self.negative_prompt[key]], dim=0)
+        else:
+            prompt = self.multimodal_encoder.generate_image(text=[text_prompt], image=image_prompt, tokenizer=self.tokenizer)
+            if do_classifier_free_guidance:
+                key = ""
+                if key not in self.negative_prompt:
+                    self.negative_prompt[key] = self.multimodal_encoder.generate_image(text=[""], tokenizer=self.tokenizer)
+                prompt = torch.cat([prompt, self.negative_prompt[key]], dim=0)
 
         return prompt