update

app.py

@@ -1,5 +1,6 @@
 import os
-from transformers import pipeline, TextIteratorStreamer
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer, pipeline
 from threading import Thread
 import gradio as gr
 import spaces
@@ -19,13 +20,70 @@ model_path = "anaspro/meta-llama-3.1-8b-inst-iraqi"
 # إذا كان فيه HF_TOKEN في البيئة
 hf_token = os.getenv("HF_TOKEN")
 
-pipe = pipeline(
-    "text-generation",
-    model=model_path,
-    torch_dtype="auto",
-    device_map="auto",
-    token=hf_token,  # إضافة التوكن إذا موجود
-)
+# استخدام ChatPipeline بدلاً من text-generation العادي
+tokenizer = AutoTokenizer.from_pretrained(model_path, token=hf_token)
+model = AutoModelForCausalLM.from_pretrained(model_path, torch_dtype="auto", device_map="auto", token=hf_token)
+
+# إنشاء chat pipeline مخصص مع streaming
+def create_chat_pipeline(tokenizer, model):
+    """إنشاء pipeline مخصص للدردشة مع chat template و streaming"""
+    def chat_generate(messages, streamer=None, **kwargs):
+        # تحويل الرسائل للـ chat template
+        if hasattr(tokenizer, 'apply_chat_template') and tokenizer.chat_template is not None:
+            prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+        else:
+            # Fallback للموديلات اللي ما عندها chat template
+            prompt = ""
+            for msg in messages:
+                if msg["role"] == "system":
+                    prompt += f"System: {msg['content']}\n"
+                elif msg["role"] == "user":
+                    prompt += f"Human: {msg['content']}\n"
+                elif msg["role"] == "assistant":
+                    prompt += f"Assistant: {msg['content']}\n"
+            prompt += "Assistant:"
+
+        inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+
+        # توليد الرد مع streaming إذا كان مطلوب
+        if streamer:
+            generation_kwargs = {
+                **inputs,
+                "max_new_tokens": kwargs.get('max_new_tokens', 512),
+                "temperature": kwargs.get('temperature', 0.7),
+                "top_p": kwargs.get('top_p', 0.9),
+                "top_k": kwargs.get('top_k', 50),
+                "repetition_penalty": kwargs.get('repetition_penalty', 1.1),
+                "do_sample": True,
+                "pad_token_id": tokenizer.eos_token_id,
+                "streamer": streamer,
+                "return_full_text": False,
+            }
+
+            # نرجع الـ thread للتشغيل
+            return generation_kwargs
+        else:
+            # للتوليد العادي بدون streaming
+            with torch.no_grad():
+                outputs = model.generate(
+                    **inputs,
+                    max_new_tokens=kwargs.get('max_new_tokens', 512),
+                    temperature=kwargs.get('temperature', 0.7),
+                    top_p=kwargs.get('top_p', 0.9),
+                    top_k=kwargs.get('top_k', 50),
+                    repetition_penalty=kwargs.get('repetition_penalty', 1.1),
+                    do_sample=True,
+                    pad_token_id=tokenizer.eos_token_id,
+                    return_dict_in_generate=True,
+                    output_scores=False,
+                )
+
+            response = tokenizer.decode(outputs.sequences[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
+            return [{"generated_text": response}]
+
+    return chat_generate
+
+pipe = create_chat_pipeline(tokenizer, model)
 
 def format_conversation_history(chat_history):
     messages = []
@@ -47,26 +105,20 @@ def generate_response(input_data, chat_history, max_new_tokens, temperature, top
     messages.extend(processed_history)
     messages.append(new_message)
 
-    # Use Llama's chat template
-    prompt_text = pipe.tokenizer.apply_chat_template(
+    # استخدام ChatPipeline المخصص مع streaming
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+    generation_kwargs = pipe(
         messages,
-        tokenize=False,
-        add_generation_prompt=True
+        streamer=streamer,
+        max_new_tokens=max_new_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        top_k=top_k,
+        repetition_penalty=repetition_penalty
     )
 
-    streamer = TextIteratorStreamer(pipe.tokenizer, skip_prompt=True, skip_special_tokens=True)
-
-    generation_kwargs = {
-        "max_new_tokens": max_new_tokens,
-        "do_sample": True,
-        "temperature": temperature,
-        "top_p": top_p,
-        "top_k": top_k,
-        "repetition_penalty": repetition_penalty,
-        "streamer": streamer,
-        "return_full_text": False,
-    }
-    thread = Thread(target=pipe, args=(prompt_text,), kwargs=generation_kwargs)
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
     thread.start()
 
     # Stream the response