Zwiększyliśmy timeout dla TextIteratorStreamer do 60 sekund.

app.py

@@ -4,7 +4,7 @@ import gradio as gr
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer
 from threading import Thread
-from queue import Queue
+from queue import Queue, Empty
 
 model_id = "meta-llama/Meta-Llama-3.1-8B"
 tokenizer = AutoTokenizer.from_pretrained(model_id, token=os.environ.get("MY_API_LLAMA_3_1"))
@@ -32,18 +32,25 @@ def generate_response(chat, kwargs):
         model = model_load_queue.get()
 
     inputs = tokenizer(chat, return_tensors="pt").to(model.device)
-    streamer = TextIteratorStreamer(tokenizer, timeout=10., skip_prompt=True, skip_special_tokens=True)
+    streamer = TextIteratorStreamer(tokenizer, timeout=60., skip_prompt=True, skip_special_tokens=True)
+
+    # Usuwamy 'seed' z kwargs, ponieważ nie jest obsługiwany przez model
+    if 'seed' in kwargs:
+        del kwargs['seed']
 
     generation_kwargs = dict(inputs, streamer=streamer, **kwargs)
     thread = Thread(target=model.generate, kwargs=generation_kwargs)
     thread.start()
 
     output = ""
-    for new_text in streamer:
-        output += new_text
-        if output.endswith("</s>"):
-            output = output[:-4]
-            break
+    try:
+        for new_text in streamer:
+            output += new_text
+            if output.endswith("</s>"):
+                output = output[:-4]
+                break
+    except Empty:
+        print("Timeout occurred during generation")
 
     return output
 
@@ -57,8 +64,7 @@ def function(prompt, history=[]):
         do_sample=True,
         temperature=0.5,
         top_p=0.95,
-        repetition_penalty=1.0,
-        seed=1337
+        repetition_penalty=1.0
     )
 
     try:
@@ -66,7 +72,7 @@ def function(prompt, history=[]):
         return output
     except Exception as e:
         print(f"Error: {str(e)}")
-        return ''
+        return 'Wystąpił błąd podczas generowania odpowiedzi.'
 
 interface = gr.ChatInterface(
     fn=function,