add pipeline

MyPipe.py

@@ -0,0 +1,32 @@
+from transformers.pipelines import PIPELINE_REGISTRY
+from transformers import Pipeline, AutoModelForCausalLM, CodeGenTokenizerFast as Tokenizer
+from PIL import Image
+from typing import Union
+
+class VQA(Pipeline):
+    def __init__(self,**kwargs):
+      # kwargs["trust_remote_code"]=True # custom architecture
+      Pipeline.__init__(self,**kwargs)
+      self.tokenizer = Tokenizer.from_pretrained("vikhyatk/moondream1")
+    def _sanitize_parameters(self, **kwargs):
+        # preprocess_params = {}
+        process = {}
+        # if "image" in kwargs :
+        #   preprocess_params["image"] = kwargs["image"]
+        if "question" in kwargs :
+          process["question"] = kwargs["question"]
+        return {}, process, {}
+
+    def preprocess(self, inputs:Union[str, Image.Image]):
+        if isinstance(inputs,str) :
+          return Image.open(inputs)
+        else:
+          return inputs
+
+    def _forward(self, inputs,question):
+        enc_image = self.model.encode_image(inputs)
+        out = self.model.answer_question(enc_image, question, self.tokenizer)
+        return out
+
+    def postprocess(self, out):
+      return out

config.json

@@ -1,15 +1,34 @@
 {
+  "_name_or_path": "vikhyatk/moondream1",
   "architectures": [
     "Moondream"
   ],
   "auto_map": {
-    "AutoConfig": "configuration_moondream.MoondreamConfig",
-    "AutoModelForCausalLM": "moondream.Moondream"
+    "AutoConfig": "vikhyatk/moondream1--configuration_moondream.MoondreamConfig",
+    "AutoModelForCausalLM": "vikhyatk/moondream1--moondream.Moondream"
+  },
+  "custom_pipelines": {
+    "visual-question-answering": {
+      "default": {
+        "model": {
+          "pt": [
+            "vikhyatk/moondream1",
+            "main"
+          ]
+        }
+      },
+      "impl": "MyPipe.VQA",
+      "pt": [
+        "AutoModelForCausalLM"
+      ],
+      "tf": [],
+      "type": "multimodal"
+    }
   },
   "model_type": "moondream1",
   "phi_config": {
     "model_type": "phi-msft"
   },
   "torch_dtype": "float32",
-  "transformers_version": "4.36.2"
+  "transformers_version": "4.35.2"
 }

generation_config.json

@@ -1,4 +1,4 @@
 {
   "_from_model_config": true,
-  "transformers_version": "4.36.2"
+  "transformers_version": "4.35.2"
 }