clip_embeddings.py

import clip  
import torch

from numpy import ndarray  
from typing import List  
from PIL import Image 

class ClipEmbeddings:

    def __init__(self, model_name: str = "ViT-B/32", device: str = "cpu"):
        
        self.device = device  # Store the specified device for model execution
        
        self.model, self.preprocess = clip.load(model_name, self.device)

    def __call__(self, docs: List[str]) -> List[ndarray]:
        # Define a method that takes a list of image file paths (docs) as input
        list_of_embeddings = []  # Create an empty list to store the image embeddings
        for image_path in docs:
            image = Image.open(image_path)  # Open and load an image from the provided path
            
            image = image.resize((224, 224))  
            # Preprocess the image and move it to the specified device
            image_input = self.preprocess(image).unsqueeze(0).to(self.device)  
            with torch.no_grad():
                # Compute the image embeddings using the CLIP model and convert 
                #them to NumPy arrays
                embeddings = self.model.encode_image(image_input).cpu().detach().numpy()
            list_of_embeddings.append(list(embeddings[0])) 
        return list_of_embeddings  

    def get_text_embeddings(self, text: str) -> List[ndarray]:
        # Define a method that takes a text string as input
        text_token = clip.tokenize(text)  # Tokenize the input text
        with torch.no_grad():
            # Compute the text embeddings using the CLIP model and convert them to NumPy arrays
            text_embeddings = self.model.encode_text(text_token).cpu().detach().numpy()
        return list(text_embeddings[0])