Duplicate from philschmid/multi-model-inference-endpoint

Co-authored-by: Philipp Schmid <philschmid@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,56 @@
+---
+tags:
+- endpoints-template
+license: apache-2.0
+---
+
+# Multi-Model GPU Inference with Hugging Face Inference Endpoints
+
+Multi-model Inference Endpoints provide a way to deploy multiple models onto the same infrastructure for a scalable and cost-effective inference. On multi-model Inference Endpoints, we load a list of models into memory, either CPU or GPU, and dynamically use them during inference time.
+
+ The following diagram shows how multi-model inference endpoints look. 
+
+![asset](mmie.png)
+
+This repository includes a [custom handler](handler.py) of a sample multi-model `EndpointHandler` implementation. This multi-model handler loads 5 different models for inference including:
+- `DistilBERT` model for `sentiment-analysis`
+- `Marian` model `translation`
+- `BART` model for `summarization`
+- `BERT` model for `token-classification`
+- `BERT` model for `text-classification`
+
+If you want to learn more about multi-model inference endpoints checkout https://www.philschmid.de/multi-model-inference-endpoints
+
+
+# Use with Inference Endpoints 
+
+Hugging Face Inference endpoints can be used with an HTTP client in any language. We will use Python and the `requests` library to send our requests. (make your you have it installed `pip install requests`)
+
+![result](inference.png)
+
+## Send requests with Pyton
+
+```python
+import json
+import requests as r
+
+ENDPOINT_URL = "" # url of your endpoint
+HF_TOKEN = "" # token of the account you deployed
+
+# define model and payload
+model_id = "facebook/bart-large-cnn"
+text = "The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct."
+request_body = {"inputs": text, "model_id": model_id}
+
+# HTTP headers for authorization
+headers= {
+    "Authorization": f"Bearer {HF_TOKEN}",
+    "Content-Type": "application/json"
+}
+
+# send request
+response = r.post(ENDPOINT_URL, headers=headers, json=request_body)
+prediction = response.json()
+
+# [{'summary_text': 'The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world.'}]
+```

handler.py

@@ -0,0 +1,41 @@
+import torch
+from typing import  Dict, List, Any
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, pipeline
+
+# check for GPU
+device = 0 if torch.cuda.is_available() else -1
+
+# multi-model list
+multi_model_list = [
+    {"model_id": "distilbert-base-uncased-finetuned-sst-2-english", "task": "text-classification"},
+    {"model_id": "Helsinki-NLP/opus-mt-en-de", "task": "translation"},
+    {"model_id": "facebook/bart-large-cnn", "task": "summarization"},
+    {"model_id": "dslim/bert-base-NER", "task": "token-classification"},
+    {"model_id": "textattack/bert-base-uncased-ag-news", "task": "text-classification"},
+]
+
+
+class EndpointHandler():
+    def __init__(self, path=""):
+        self.multi_model={}
+        # load all the models onto device
+        for model in multi_model_list:
+            self.multi_model[model["model_id"]] = pipeline(model["task"], model=model["model_id"], device=device)
+
+    def __call__(self, data: Any) -> List[List[Dict[str, float]]]:
+        # deserialize incomin request
+        inputs = data.pop("inputs", data)
+        parameters = data.pop("parameters", None)
+        model_id = data.pop("model_id", None)
+        
+        # check if model_id is in the list of models
+        if model_id is None or model_id not in self.multi_model:
+            raise ValueError(f"model_id: {model_id} is not valid. Available models are: {list(self.multi_model.keys())}")
+
+        # pass inputs with all kwargs in data
+        if parameters is not None:
+            prediction = self.multi_model[model_id](inputs, **parameters)
+        else:
+            prediction = self.multi_model[model_id](inputs)
+        # postprocess the prediction
+        return prediction