Update app.py

app.py

@@ -98,17 +98,35 @@ def predict_chat(message: str, history: list):
                 prompt_input += f"Assistant: {msg['content']}\n"
         prompt_input += "Assistant:"
 
-        # FIXED: Use max_tokens instead of max_new_tokens for ctransformers
-        for token in model.generate(
-            prompt_input,
-            max_tokens=MAX_NEW_TOKENS,  # Changed from max_new_tokens
-            temperature=TEMPERATURE,
-            top_k=TOP_K,
-            top_p=TOP_P,
-            sample=DO_SAMPLE,  # Changed from do_sample
-            repetition_penalty=1.1,
-            stop=["User:", "\nUser", "\n#", "\n##", "<|endoftext|>"] 
-        ):
+        # FIXED: Use the correct ctransformers method - call model() directly for streaming
+        try:
+            for token in model(
+                prompt_input,
+                max_new_tokens=MAX_NEW_TOKENS,
+                temperature=TEMPERATURE,
+                top_k=TOP_K,
+                top_p=TOP_P,
+                do_sample=DO_SAMPLE,
+                repetition_penalty=1.1,
+                stop=["User:", "\nUser", "\n#", "\n##", "<|endoftext|>"],
+                stream=True
+            ):
+                generated_text += token
+                yield generated_text
+        except Exception as e:
+            print(f"Error in GGUF generation: {e}")
+            # Fallback to non-streaming generation
+            output = model(
+                prompt_input,
+                max_new_tokens=MAX_NEW_TOKENS,
+                temperature=TEMPERATURE,
+                top_k=TOP_K,
+                top_p=TOP_P,
+                do_sample=DO_SAMPLE,
+                repetition_penalty=1.1,
+                stop=["User:", "\nUser", "\n#", "\n##", "<|endoftext|>"]
+            )
+            yield output
             generated_text += token
             yield generated_text