Update app.py

app.py

@@ -17,7 +17,7 @@ def detokenize(tokens, model_name="lmsys/vicuna-7b-v1.5"):
   string = tokenizer.decode(tokens[0])
   return string
 
-def emulate(text, ttft=1000, out_tps=10, in_tps=None, model_name="lmsys/vicuna-7b-v1.5", n=10):
+def emulate(text, ttft=1000, out_tps=10, in_tps=None, model_name="lmsys/vicuna-7b-v1.5", n=10, offset=0):
   # get TTFT
   if in_tps is not None and ttft is not None:
     print("both TTFT & Input Tokens per second specified, using TTFT")
@@ -45,8 +45,7 @@ def emulate(text, ttft=1000, out_tps=10, in_tps=None, model_name="lmsys/vicuna-7
   print(f'Starting...\n\n')
   start_time = time.time()
   # Delay by ttft
-  # sleep_ms(max(ttft-1000,0))
-  sleep_ms(ttft)
+  sleep_ms(max(ttft-offset*1000,0))
   ttft_time = time.time()
 
   # yield text
@@ -87,6 +86,8 @@ demo = gr.Interface(
         gr.Slider(0, 1000, value=10, label="Output Tokens per Second"),
     ],
     outputs="text",
+    additional_inputs=[gr.Slider(minimum=0, maximum=2, step=0.1, label="TTFT Offset (S)")],
+
     # live=True
 )