fix throughput measure

app.py

@@ -1,7 +1,6 @@
 import gradio as gr
 
 import json
-import math
 from backend import get_message_single, get_message_spam, send_single, send_spam, tokenizer
 from defaults import (
     ADDRESS_BETTERTRANSFORMER,
@@ -10,7 +9,6 @@ from defaults import (
     defaults_bt_spam,
     defaults_vanilla_single,
     defaults_vanilla_spam,
-    BATCH_SIZE,
 )
 
 import datasets
@@ -22,17 +20,6 @@ def dispatch_single(input_model_single, address_input_vanilla, address_input_bet
     
     return result_vanilla, result_bettertransformer
 
-def dispatch_spam(input_n_spam, address_input_vanilla, address_input_bettertransformer):
-    input_n_spam = int(input_n_spam)
-    assert input_n_spam <= len(data)
-
-    inp = data.shuffle().select(range(input_n_spam))
-
-    result_vanilla = send_spam(inp, address_input_vanilla)
-    result_bettertransformer = send_spam(inp, address_input_bettertransformer)
-    
-    return result_vanilla, result_bettertransformer
-
 def dispatch_spam_artif(input_n_spam_artif, sequence_length, padding_ratio, address_input_vanilla, address_input_bettertransformer):    
     sequence_length = int(sequence_length)
     input_n_spam_artif = int(input_n_spam_artif)
@@ -44,7 +31,6 @@ def dispatch_spam_artif(input_n_spam_artif, sequence_length, padding_ratio, addr
 
     inp_tokens[0] = 101
     inp_tokens[- n_pads - 1] = 102
-    #inp_tokens = inp_tokens.unsqueeze(0).repeat(BATCH_SIZE, 1)
     
     attention_mask = torch.zeros((sequence_length,), dtype=torch.int64)
     attention_mask[:- n_pads] = 1
@@ -63,7 +49,7 @@ def dispatch_spam_artif(input_n_spam_artif, sequence_length, padding_ratio, addr
     result_bettertransformer = send_spam(input_dataset, address_input_bettertransformer)
 
     return result_vanilla, result_bettertransformer
-  
+
 TTILE_IMAGE = """
 <div
     style="

backend.py

@@ -6,11 +6,11 @@ from defaults import (
     HEADERS,
     MODEL_NAME,
 )
-from utils import ElapsedFuturesSession
+from requests_futures.sessions import FuturesSession
 
 from transformers import AutoTokenizer
 
-import numpy as np
+import time
 
 RETURN_MESSAGE_SINGLE = """
 Inference statistics:
@@ -68,18 +68,20 @@ def get_message_spam(
     )
 
 
-SESSION = ElapsedFuturesSession()
+SESSION = FuturesSession()
 
 def send_single(input_model_vanilla, address: str):
     assert address in [ADDRESS_VANILLA, ADDRESS_BETTERTRANSFORMER]
 
     # should not take more than 10 s, so timeout if that's the case
+    start = time.time()
     promise = SESSION.post(
         address, headers=HEADERS, data=input_model_vanilla.encode("utf-8"), timeout=10
     )
 
     try:
         response = promise.result()  # resolve ASAP
+        end = time.time()
     except Exception as e:
         return f"{e}"
 
@@ -89,7 +91,7 @@ def send_single(input_model_vanilla, address: str):
     prediction = response_text[0]
     inf_latency = response_text[1]
     peak_gpu_memory = response_text[2]
-    end_to_end_latency = response.elapsed
+    end_to_end_latency = round((end - start) * 1e3, 2)
 
     return get_message_single(
         status, prediction, inf_latency, peak_gpu_memory, end_to_end_latency
@@ -99,8 +101,6 @@ def send_single(input_model_vanilla, address: str):
 def send_spam(inp, address: str):
     assert address in [ADDRESS_VANILLA, ADDRESS_BETTERTRANSFORMER]
 
-    # data = "this is positive lol"  #TODO: use dynamic data with padding
-    max_resolution_time = 0
     mean_inference_latency = 0
     mean_peak_gpu_memory = 0
 
@@ -113,6 +113,7 @@ def send_spam(inp, address: str):
 
     n_inputs = len(inp)
 
+    start = time.time()
     for i in range(n_inputs):
         input_data = inp[i]["sentence"].encode("utf-8")
 
@@ -121,18 +122,21 @@ def send_spam(inp, address: str):
             SESSION.post(address, headers=HEADERS, data=input_data, timeout=15)
         )
 
+    # to measure throughput first
+    end = 0
     for promise in promises:
         try:
             response = promise.result()  # resolve ASAP
         except Exception as e:
             return f"{e}"
+        
+        end = max(time.time(), end)
 
+    # then other metrics
+    for promise in promises:
         response = promise.result()
-
         response_text = json.loads(response.text)
 
-        max_resolution_time = max(max_resolution_time, response.elapsed)
-
         mean_inference_latency += response_text[1]
         mean_peak_gpu_memory += response_text[2]
         n_pads += response_text[3]
@@ -140,7 +144,7 @@ def send_spam(inp, address: str):
         sequence_length += response_text[5]
         effective_batch_size += response_text[6]
 
-    throughput = n_inputs / (max_resolution_time * 1e-3)
+    throughput = n_inputs / (end - start)
     mean_padding_ratio = f"{n_pads / n_elems * 100:.2f}"
     mean_sequence_length = sequence_length / n_inputs
     effective_batch_size = effective_batch_size / n_inputs

utils.py

@@ -1,26 +0,0 @@
-import time
-
-from requests_futures.sessions import FuturesSession
-
-
-class ElapsedFuturesSession(FuturesSession):
-    def request(self, method, url, hooks=None, *args, **kwargs):
-        start = time.time()
-        if hooks is None:
-            hooks = {}
-
-        def timing(r, *args, **kwargs):
-            r.elapsed = round((time.time() - start) * 1000, 2)
-
-        try:
-            if isinstance(hooks["response"], (list, tuple)):
-                # needs to be first so we don't time other hooks execution
-                hooks["response"].insert(0, timing)
-            else:
-                hooks["response"] = [timing, hooks["response"]]
-        except KeyError:
-            hooks["response"] = timing
-
-        return super(ElapsedFuturesSession, self).request(
-            method, url, hooks=hooks, *args, **kwargs
-        )