chore: increase max_new_tokens parameter to 4096 across chat model and inference functions for improved response generation

src/app.py

@@ -240,7 +240,7 @@ else:
             full_response = ""
             
             try:
-                for token in chat_model.generate_streaming(max_new_tokens=512):
+                for token in chat_model.generate_streaming(max_new_tokens=4096):
                     full_response += token
                     response_placeholder.markdown(full_response)
                 

src/backend/chat_model.py

@@ -82,7 +82,7 @@ class ChatModel:
     
     def generate_streaming(
         self,
-        max_new_tokens: int = 512,
+        max_new_tokens: int = 4096,
         temperature: Optional[float] = None,
         top_p: Optional[float] = None,
         top_k: Optional[int] = None,
@@ -116,7 +116,7 @@ class ChatModel:
     
     def generate(
         self,
-        max_new_tokens: int = 512,
+        max_new_tokens: int = 4096,
         temperature: Optional[float] = None,
         top_p: Optional[float] = None,
         top_k: Optional[int] = None,
@@ -148,7 +148,7 @@ class ChatModel:
     def chat(
         self,
         user_message: str,
-        max_new_tokens: int = 512,
+        max_new_tokens: int = 4096,
         temperature: Optional[float] = None,
         streaming: bool = False,
     ) -> Union[str, Iterator[str]]:

src/backend/inference.py

@@ -34,7 +34,7 @@ def _build_generation_kwargs(
 def generate_streaming(
     pipeline: Pipeline,
     prompt: Union[str, List[Message]],
-    max_new_tokens: int = 512,
+    max_new_tokens: int = 4096,
     temperature: Optional[float] = None,
     top_p: Optional[float] = None,
     top_k: Optional[int] = None,
@@ -112,7 +112,7 @@ def generate_streaming(
 def generate_simple(
     pipeline: Pipeline,
     prompt: Union[str, List[Message]],
-    max_new_tokens: int = 512,
+    max_new_tokens: int = 4096,
     temperature: Optional[float] = None,
     top_p: Optional[float] = None,
     top_k: Optional[int] = None,