asnyc trial

app.py

@@ -18,9 +18,9 @@ max_pixels = 1280 * 28 * 28
 processor: Qwen2VLProcessor = AutoProcessor.from_pretrained(model_path, min_pixels=min_pixels, max_pixels=max_pixels)
 
 @GPU
-def infer(inputs: BatchFeature, **kwargs):
+async def infer(inputs: BatchFeature, **kwargs):
     inputs = inputs.to("cuda")
-    Thread(target=model.generate, kwargs=kwargs).start()
+    model.generate(**inputs, **kwargs)
 
 def respond(
     message,
@@ -45,13 +45,12 @@ def respond(
     )
     streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)
     kwargs = dict(
-        inputs=inputs,
         streamer=streamer,
         max_new_tokens=max_tokens,
         temperature=temperature,
         top_p=top_p,
     )
-    infer(**kwargs)
+    infer(inputs, **kwargs)
     response = ""
     for token in streamer:
         response += token