wip

.gitignore

@@ -1 +1,5 @@
 .venv
+vision.pt
+text.pt
+*pnnx*
+*_ncnn.py

export.py

@@ -1,3 +1,4 @@
+import json
 from PIL import Image
 import requests
 from transformers import CLIPProcessor, CLIPModel
@@ -7,6 +8,7 @@ import torch
 import numpy as np
 import platform
 import sys
+import os
 
 processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
 
@@ -73,6 +75,8 @@ def convert_coreml():
     coreml_model = ct.convert(traced_text_model, inputs=[ct.TensorType(shape=inputs.data['input_ids'].shape), ct.TensorType(shape=inputs.data['attention_mask'].shape)])
     coreml_model.save('text.mlpackage')
 
+# convert_coreml()
+
 def infer_coreml():
     coreml_vision_model = ct.models.MLModel('vision.mlpackage')
     coreml_text_model = ct.models.MLModel('text.mlpackage')
@@ -95,7 +99,6 @@ def convert_onnx():
 
 # convert_onnx()
 
-
 def infer_onnx():
     import onnxruntime as ort
 
@@ -129,8 +132,7 @@ def infer_onnx():
 
     print("similarity:", logits_per_text.item())
 
-infer_onnx()
-
+# infer_onnx()
 
 def convert_openvino():
     import openvino as ov
@@ -162,13 +164,49 @@ def infer_openvino():
 
 # infer_openvino()
 
-# convert_coreml()
 
-# Create an ONNX Runtime session
-# session = ort.InferenceSession(model_path)
-# outputs = session.run(None, np_inputs)
+def export_ncnn():
+    traced_vision_model.save(f"vision.pt")
+    input_shape_str = json.dumps(list(inputs.data['pixel_values'].shape)).replace(" ", "")
+    os.system(f"pnnx vision.pt 'inputshape={input_shape_str}'")
+
+    traced_text_model.save(f"text.pt")
+    input_shape_str = json.dumps(list(inputs.data['input_ids'].shape)).replace(" ", "")
+    input_shape2_str = json.dumps(list(inputs.data['attention_mask'].shape)).replace(" ", "")
+    os.system(f"pnnx text.pt 'inputshape={input_shape_str}i64,{input_shape2_str}i64'")
+
+export_ncnn()
+
+def infer_ncnn():
+    import ncnn
+    
+    vision_extractor = ncnn.Net()
+    vision_extractor.load_param("vision.param")
+    vision_extractor.load_model("vision.bin")
+
+    text_extractor = ncnn.Net()
+    text_extractor.load_param("text.param")
+    text_extractor.load_model("text.bin")
+
+    vision_mat = ncnn.Mat(inputs.data['pixel_values'].numpy())
+    text_input_ids_mat = ncnn.Mat(inputs.data['input_ids'].numpy())
+    text_attention_mask_mat = ncnn.Mat(inputs.data['attention_mask'].numpy())
+
+    vision_extractor.input(vision_extractor.input_names()[0], vision_mat)
+    text_extractor.input(text_extractor.input_names()[0], text_input_ids_mat)
+    text_extractor.input(text_extractor.input_names()[1], text_attention_mask_mat)
+
+    image_embeds = vision_extractor.extract("out0")
+    text_embeds = text_extractor.extract("out0")
+
+    logits_per_text = text_embeds @ image_embeds.T
+
+    print("similarity:", logits_per_text[0])
+
+infer_ncnn()
 
-# outputs = ptmodel(**inputs)
-# logits_per_image = outputs.logits_per_image # this is the image-text similarity score
-# probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
-# print(logits_per_image)
+def infer_torch():
+    outputs = ptmodel(**inputs)
+    logits_per_image = outputs.logits_per_image # this is the image-text similarity score
+    probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities
+    print(probs)