Update app.py

app.py

@@ -1,6 +1,8 @@
 import os
 from huggingface_hub import login
 from transformers import BlipProcessor, BlipForConditionalGeneration
+from PIL import Image
+import pytesseract
 
 
 
@@ -28,17 +30,25 @@ model.to(device)
 
 @spaces.GPU(duration=300)
 def generate_caption_and_image(image):
+    img = image.convert("RGB")
+
+    
+    extracted_text = pytesseract.image_to_string(img)
     # Process the image
     raw_image = image.convert("RGB")
+      
+
     
     # Generate caption
     inputs = processor(raw_image, return_tensors="pt", padding=True, truncation=True, max_length=250)
     inputs = {key: val.to(device) for key, val in inputs.items()}
     out = model.generate(**inputs)
     caption = processor.decode(out[0], skip_special_tokens=True)
+    prompt = f"Create a highly realistic design of a clothing item based on the following description: 'The design should incorporate elements from the extracted text: {extracted_text}, along with the caption: {caption}. The clothing should look realistic, modern, and stylish. Use high-quality fabric textures and realistic lighting to give the design a lifelike appearance. The colors, patterns, and materials should reflect the essence of the caption and extracted text.'"
+
 
     # Generate image based on the caption
-    generated_image = pipe(caption).images[0]
+    generated_image = pipe(prompt).images[0]
 
     return caption, generated_image