Use CLIP forward embeddings

app.py

@@ -109,11 +109,11 @@ PRESETS = {
 DEFAULT_PRESET = "Object names"
 
 CLIP_CODE = """# CLIP makes text and images comparable by putting them in one space.
-text_inputs = processor(text=phrases, padding=True, truncation=True, return_tensors="pt")
-image_inputs = processor(images=images, return_tensors="pt")
+inputs = processor(text=phrases, images=images, padding=True, truncation=True, return_tensors="pt")
+outputs = model(**inputs)
 
-text_features = model.get_text_features(**text_inputs)
-image_features = model.get_image_features(**image_inputs)
+text_features = outputs.text_embeds
+image_features = outputs.image_embeds
 
 # Normalize first, then dot product equals cosine similarity.
 text_features = text_features / text_features.norm(dim=-1, keepdim=True)
@@ -338,17 +338,18 @@ def to_device(batch, device):
 def encode_clip(texts, images):
     model, processor, device = load_clip()
 
-    text_inputs = processor(
+    inputs = processor(
         text=texts,
+        images=images,
         padding=True,
         truncation=True,
         return_tensors="pt",
     )
-    image_inputs = processor(images=images, return_tensors="pt")
 
     with torch.inference_mode():
-        text_features = model.get_text_features(**to_device(text_inputs, device))
-        image_features = model.get_image_features(**to_device(image_inputs, device))
+        outputs = model(**to_device(inputs, device))
+        text_features = outputs.text_embeds
+        image_features = outputs.image_embeds
         text_features = text_features / text_features.norm(dim=-1, keepdim=True)
         image_features = image_features / image_features.norm(dim=-1, keepdim=True)
         cosine = text_features @ image_features.T