update app

app.py

@@ -59,18 +59,41 @@ def generate_stream(prompt, max_new_tokens=128):
     #     text = tokenizer.decode(tok, skip_special_tokens=True)
     #     if text.strip():
     #         yield text
-            
-    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
-    generation_kwargs = dict(
-        **inputs,
-        max_new_tokens=max_new_tokens,
-        eos_token_id=tokenizer.eos_token_id,
-        do_sample=True,
-        temperature=0.7,
-        streamer=streamer
-    )
 
-    thread = threading.Thread(target=model.generate, kwargs=generation_kwargs)
+    # streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    # generation_kwargs = dict(
+    #     **inputs,
+    #     max_new_tokens=max_new_tokens,
+    #     eos_token_id=tokenizer.eos_token_id,
+    #     do_sample=True,
+    #     temperature=0.7,
+    #     streamer=streamer
+    # )
+
+    # thread = threading.Thread(target=model.generate, kwargs=generation_kwargs)
+    # thread.start()
+    
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    def run_generation():
+        try:
+            model.generate(
+                **inputs,
+                max_new_tokens=max_new_tokens,
+                do_sample=True,
+                temperature=0.7,
+                top_p=0.9,
+                streamer=streamer,
+                eos_token_id=tokenizer.eos_token_id,
+                pad_token_id=tokenizer.pad_token_id,
+                use_cache=True,
+            )
+        except Exception as e:
+            # simpan error agar bisa dikembalikan ke client setelah streamer selesai
+            # error_container.append(str(e))
+            pass
+
+    thread = threading.Thread(target=run_generation, daemon=True)
     thread.start()
 
     for token in streamer: