update

app.py

@@ -1,6 +1,8 @@
 import gradio as gr
 
-from backend import get_message_single, get_message_spam, send_single, send_spam
+import json
+import math
+from backend import get_message_single, get_message_spam, send_single, send_spam, tokenizer
 from defaults import (
     ADDRESS_BETTERTRANSFORMER,
     ADDRESS_VANILLA,
@@ -8,8 +10,60 @@ from defaults import (
     defaults_bt_spam,
     defaults_vanilla_single,
     defaults_vanilla_spam,
+    BATCH_SIZE,
 )
 
+import datasets
+import torch
+
+def dispatch_single(input_model_single, address_input_vanilla, address_input_bettertransformer):
+    result_vanilla = send_single(input_model_single, address_input_vanilla)
+    result_bettertransformer = send_single(input_model_single, address_input_bettertransformer)
+    
+    return result_vanilla, result_bettertransformer
+
+def dispatch_spam(input_n_spam, address_input_vanilla, address_input_bettertransformer):
+    input_n_spam = int(input_n_spam)
+    assert input_n_spam <= len(data)
+
+    inp = data.shuffle().select(range(input_n_spam))
+
+    result_vanilla = send_spam(inp, address_input_vanilla)
+    result_bettertransformer = send_spam(inp, address_input_bettertransformer)
+    
+    return result_vanilla, result_bettertransformer
+
+def dispatch_spam_artif(input_n_spam_artif, sequence_length, padding_ratio, address_input_vanilla, address_input_bettertransformer):    
+    sequence_length = int(sequence_length)
+    input_n_spam_artif = int(input_n_spam_artif)
+    
+    inp_tokens = torch.randint(tokenizer.vocab_size - 1, (sequence_length,)) + 1
+
+    n_pads = max(int(padding_ratio * len(inp_tokens)), 1)
+    inp_tokens[- n_pads:] = 0
+
+    inp_tokens[0] = 101
+    inp_tokens[- n_pads - 1] = 102
+    #inp_tokens = inp_tokens.unsqueeze(0).repeat(BATCH_SIZE, 1)
+    
+    attention_mask = torch.zeros((sequence_length,), dtype=torch.int64)
+    attention_mask[:- n_pads] = 1
+    
+    str_input = json.dumps({
+        "input_ids": inp_tokens.cpu().tolist(),
+        "attention_mask": attention_mask.cpu().tolist(),
+        "pre_tokenized": True,
+    })
+    
+    input_dataset = datasets.Dataset.from_dict(
+        {"sentence": [str_input for _ in range(input_n_spam_artif)]}
+    )
+    
+    result_vanilla = send_spam(input_dataset, address_input_vanilla)
+    result_bettertransformer = send_spam(input_dataset, address_input_bettertransformer)
+
+    return result_vanilla, result_bettertransformer
+  
 TTILE_IMAGE = """
 <div
     style="
@@ -34,7 +88,7 @@ TITLE = """
         font-size: 2.2rem;
     "
 >
-<h1 style="font-weight: 700; margin-bottom: 10px; margin-top: 10px;">
+<h1 style="font-weight: 500; margin-bottom: 10px; margin-top: 10px;">
     Speed up your inference and support more workload with PyTorch's BetterTransformer 🤗
 </h1>
 </div>
@@ -67,98 +121,76 @@ with gr.Blocks() as demo:
     with gr.Row():
         with gr.Column(scale=50):
             gr.Markdown("### Vanilla Transformers + TorchServe")
+        with gr.Column(scale=50):
+            gr.Markdown("### BetterTransformer + TorchServe")
+        
+    address_input_vanilla = gr.Textbox(
+        max_lines=1, label="ip vanilla", value=ADDRESS_VANILLA, visible=False
+    )
 
-            address_input_vanilla = gr.Textbox(
-                max_lines=1, label="ip vanilla", value=ADDRESS_VANILLA, visible=False
-            )
+    address_input_bettertransformer = gr.Textbox(
+        max_lines=1,
+        label="ip bettertransformer",
+        value=ADDRESS_BETTERTRANSFORMER,
+        visible=False,
+    )
 
-            input_model_vanilla = gr.Textbox(
-                max_lines=1,
-                label="Text",
-                value="Expectations were low, enjoyment was high",
-            )
+    input_model_single = gr.Textbox(
+        max_lines=1,
+        label="Text",
+        value="Expectations were low, enjoyment was high",
+    )
 
-            btn_single_vanilla = gr.Button("Send single text request")
+    btn_single = gr.Button("Send single text request")
+    with gr.Row():
+        with gr.Column(scale=50):
             output_single_vanilla = gr.Markdown(
                 label="Output single vanilla",
                 value=get_message_single(**defaults_vanilla_single),
             )
-            with gr.Column():
-                with gr.Column(scale=40):
-                    input_n_inputs_vanilla = gr.Textbox(
-                        max_lines=1,
-                        label="Number of inputs",
-                        value=8,
-                    )
-                with gr.Column(scale=60):
-                    gr.Markdown("")
-                    btn_spam_vanilla = gr.Button(
-                        "Spam text requests (from sst2 validation set)"
-                    )
-
-            output_spam_vanilla = gr.Markdown(
-                label="Output spam vanilla",
-                value=get_message_spam(**defaults_vanilla_spam),
-            )
-
-            btn_single_vanilla.click(
-                fn=send_single,
-                inputs=[input_model_vanilla, address_input_vanilla],
-                outputs=output_single_vanilla,
-            )
-            btn_spam_vanilla.click(
-                fn=send_spam,
-                inputs=[address_input_vanilla],
-                outputs=output_spam_vanilla,
-            )
-
         with gr.Column(scale=50):
-            gr.Markdown("### BetterTransformer + TorchServe")
-
-            address_input_bettertransformer = gr.Textbox(
-                max_lines=1,
-                label="ip bettertransformer",
-                value=ADDRESS_BETTERTRANSFORMER,
-                visible=False,
-            )
-
-            input_model_bettertransformer = gr.Textbox(
-                max_lines=1,
-                label="Text",
-                value="Expectations were low, enjoyment was high",
-            )
-
-            btn_single_bt = gr.Button("Send single text request")
             output_single_bt = gr.Markdown(
                 label="Output single bt", value=get_message_single(**defaults_bt_single)
             )
 
-            with gr.Row():
-                with gr.Column(scale=40):
-                    input_n_inputs_bt = gr.Textbox(
-                        max_lines=1,
-                        label="Number of inputs",
-                        value=8,
-                    )
-                with gr.Column(scale=60):
-                    gr.Markdown("")
-                    btn_spam_bt = gr.Button("Spam text requests (from sst2 validation set)")
-            
-            output_spam_bt = gr.Markdown(
+    btn_single.click(
+        fn=dispatch_single,
+        inputs=[input_model_single, address_input_vanilla, address_input_bettertransformer],
+        outputs=[output_single_vanilla, output_single_bt],
+    )
+
+    input_n_spam_artif = gr.Number(
+        label="Number of inputs to send",
+        value=8,
+    )
+    sequence_length = gr.Number(
+        label="Sequence length (in tokens)",
+        value=128,
+    )
+    padding_ratio = gr.Number(
+        label="Padding ratio",
+        value=0.5,
+    )
+    btn_spam_artif = gr.Button(
+        "Spam text requests (using artificial data)"
+    )
+    with gr.Row():
+        with gr.Column(scale=50):
+            output_spam_vanilla_artif = gr.Markdown(
+                label="Output spam vanilla",
+                value=get_message_spam(**defaults_vanilla_spam),
+            )
+        with gr.Column(scale=50):
+            output_spam_bt_artif = gr.Markdown(
                 label="Output spam bt", value=get_message_spam(**defaults_bt_spam)
             )
 
-            btn_single_bt.click(
-                fn=send_single,
-                inputs=[input_model_bettertransformer, address_input_bettertransformer],
-                outputs=output_single_bt,
-            )
+    btn_spam_artif.click(
+        fn=dispatch_spam_artif,
+        inputs=[input_n_spam_artif, sequence_length, padding_ratio, address_input_vanilla, address_input_bettertransformer],
+        outputs=[output_spam_vanilla_artif, output_spam_bt_artif],
+    )
 
-            btn_spam_bt.click(
-                fn=send_spam,
-                inputs=[address_input_bettertransformer],
-                outputs=output_spam_bt,
-            )
 
 demo.queue(concurrency_count=1)
-demo.launch()
+demo.launch()

backend.py

@@ -1,16 +1,16 @@
 import json
 
-from datasets import load_dataset
-
 from defaults import (
     ADDRESS_BETTERTRANSFORMER,
     ADDRESS_VANILLA,
     HEADERS,
-    SPAM_N_REQUESTS,
+    MODEL_NAME,
 )
 from utils import ElapsedFuturesSession
 
-data = load_dataset("glue", "sst2", split="validation")
+from transformers import AutoTokenizer
+
+import numpy as np
 
 RETURN_MESSAGE_SINGLE = """
 Inference statistics:
@@ -26,19 +26,20 @@ Inference statistics:
 RETURN_MESSAGE_SPAM = (
     """
 Processing """
-    + f"{SPAM_N_REQUESTS}"
-    + """ inputs sent asynchronously. Grab a coffee.
+    + "NUMBER REQ" + """ inputs sent asynchronously. Grab a coffee.
 
 Inference statistics:
 
-* Promise resolution time: {0} ms
+* Throughput: {0} samples/s
 * Mean inference latency (preprocessing/forward/postprocessing): {1} ms
 * Mean peak GPU memory: {2} MB
 * Mean padding ratio: {3} %
 * Mean sequence length: {4} tokens
+* Effective mean batch size: {5}
 """
 )
 
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 
 def get_message_single(
     status, prediction, inf_latency, peak_gpu_memory, end_to_end_latency, **kwargs
@@ -49,25 +50,26 @@ def get_message_single(
 
 
 def get_message_spam(
-    resolution_time,
+    throughput,
     mean_inference_latency,
     mean_peak_gpu_memory,
     mean_padding_ratio,
     mean_sequence_length,
+    effective_batch_size,
     **kwargs,
 ):
     return RETURN_MESSAGE_SPAM.format(
-        resolution_time,
+        throughput,
         mean_inference_latency,
         mean_peak_gpu_memory,
         mean_padding_ratio,
         mean_sequence_length,
+        effective_batch_size,
     )
 
 
 SESSION = ElapsedFuturesSession()
 
-
 def send_single(input_model_vanilla, address: str):
     assert address in [ADDRESS_VANILLA, ADDRESS_BETTERTRANSFORMER]
 
@@ -94,26 +96,24 @@ def send_single(input_model_vanilla, address: str):
     )
 
 
-def send_spam(address: str):
+def send_spam(inp, address: str):
     assert address in [ADDRESS_VANILLA, ADDRESS_BETTERTRANSFORMER]
 
     # data = "this is positive lol"  #TODO: use dynamic data with padding
-
-    assert SPAM_N_REQUESTS <= len(data)
-
-    inp = data.shuffle().select(range(SPAM_N_REQUESTS))
-
-    resolution_time = 0
+    max_resolution_time = 0
     mean_inference_latency = 0
     mean_peak_gpu_memory = 0
 
     n_pads = 0
     n_elems = 0
     sequence_length = 0
+    effective_batch_size = 0
 
     promises = []
 
-    for i in range(SPAM_N_REQUESTS):
+    n_inputs = len(inp)
+
+    for i in range(n_inputs):
         input_data = inp[i]["sentence"].encode("utf-8")
 
         # should not take more than 15 s, so timeout if that's the case
@@ -131,25 +131,29 @@ def send_spam(address: str):
 
         response_text = json.loads(response.text)
 
-        resolution_time = max(resolution_time, response.elapsed)
+        max_resolution_time = max(max_resolution_time, response.elapsed)
 
         mean_inference_latency += response_text[1]
         mean_peak_gpu_memory += response_text[2]
         n_pads += response_text[3]
         n_elems += response_text[4]
         sequence_length += response_text[5]
+        effective_batch_size += response_text[6]
 
+    throughput = n_inputs / (max_resolution_time * 1e-3)
     mean_padding_ratio = f"{n_pads / n_elems * 100:.2f}"
-    mean_sequence_length = sequence_length / SPAM_N_REQUESTS
+    mean_sequence_length = sequence_length / n_inputs
+    effective_batch_size = effective_batch_size / n_inputs
 
-    resolution_time = round(resolution_time, 2)
-    mean_inference_latency = round(mean_inference_latency / SPAM_N_REQUESTS, 2)
-    mean_peak_gpu_memory = round(mean_peak_gpu_memory / SPAM_N_REQUESTS, 2)
+    throughput = round(throughput, 2)
+    mean_inference_latency = round(mean_inference_latency / n_inputs, 2)
+    mean_peak_gpu_memory = round(mean_peak_gpu_memory / n_inputs, 2)
 
     return get_message_spam(
-        resolution_time,
+        throughput,
         mean_inference_latency,
         mean_peak_gpu_memory,
         mean_padding_ratio,
         mean_sequence_length,
+        effective_batch_size,
     )

defaults.py

@@ -15,24 +15,26 @@ defaults_bt_single = {
 }
 
 defaults_vanilla_spam = {
-    "resolution_time": 2996.35,
+    "throughput": 20,
     "mean_inference_latency": 29.69,
     "mean_peak_gpu_memory": 3620.9,
     "mean_padding_ratio": 35.26,
     "mean_sequence_length": 39.395,
+    "effective_batch_size": 8,
 }
 
 defaults_bt_spam = {
-    "resolution_time": 2996.35,
+    "throughput": 20,
     "mean_inference_latency": 29.69,
     "mean_peak_gpu_memory": 3620.9,
     "mean_padding_ratio": 35.26,
     "mean_sequence_length": 39.395,
+    "effective_batch_size": 8,
 }
 
-SPAM_N_REQUESTS = 200
 BATCH_SIZE = 8  # fixed!
 
 HEADERS = {"Content-Type": "text/plain"}
 ADDRESS_VANILLA = "http://3.83.142.46:8080/predictions/my_tc"
 ADDRESS_BETTERTRANSFORMER = "http://3.95.136.2:8080/predictions/my_tc"
+MODEL_NAME = "distilbert-base-uncased-finetuned-sst-2-english"