Embeded generator

app.py

@@ -18,9 +18,14 @@ max_pixels = 1280 * 28 * 28
 processor: Qwen2VLProcessor = AutoProcessor.from_pretrained(model_path, min_pixels=min_pixels, max_pixels=max_pixels)
 
 @GPU
-async def infer(inputs: BatchFeature, **kwargs):
+def infer(inputs: BatchFeature, streamer, kwargs: dict):
     inputs = inputs.to("cuda")
-    model.generate(**inputs, **kwargs)
+    thread = Thread(target=model.generate, kwargs={**inputs, **kwargs})
+    thread.start()
+    response = ""
+    for token in streamer:
+        response += token
+        yield response
 
 def respond(
     message,
@@ -50,10 +55,7 @@ def respond(
         temperature=temperature,
         top_p=top_p,
     )
-    infer(inputs, **kwargs)
-    response = ""
-    for token in streamer:
-        response += token
+    for response in infer(inputs, streamer, kwargs):
         yield response
 
 app = ChatInterface(