fix work with agent api mode

src/api/routes.py

@@ -1,6 +1,6 @@
 import asyncio
 import json
-import threading
+import logging
 
 from fastapi import APIRouter, HTTPException, Request
 from fastapi.responses import StreamingResponse
@@ -9,6 +9,9 @@ from src.core.config import settings
 from src.core.engine import engine
 from src.utils.helpers import get_clean_text
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 router = APIRouter()
 
 
@@ -17,13 +20,33 @@ async def chat_completions(request: Request):
     if not engine.llm:
         raise HTTPException(status_code=500, detail="Model not loaded")
 
-    data = await request.json()
+    try:
+        data = await request.json()
+    except Exception:
+        raise HTTPException(status_code=400, detail="Invalid JSON")
+
+    logger.info(
+        f"API Request received. Model: {data.get('model')}, Stream: {data.get('stream', True)}"
+    )
+
     messages = [
         {"role": m.get("role", "user"), "content": get_clean_text(m.get("content"))}
         for m in data.get("messages", [])
     ]
+
     max_tokens = data.get("max_tokens", settings.DEFAULT_MAX_TOKENS)
     temperature = data.get("temperature", settings.DEFAULT_TEMP)
+    top_p = data.get("top_p", 0.95)
+
+    stop = data.get("stop", [])
+    if isinstance(stop, str):
+        stop = [stop]
+
+    default_stops = ["<|im_end|>", "<|endoftext|>"]
+    for s in default_stops:
+        if s not in stop:
+            stop.append(s)
+
     stream_req = data.get("stream", True)
 
     # --- Логика Streaming ---
@@ -33,28 +56,49 @@ async def chat_completions(request: Request):
 
         def worker():
             try:
-                for chunk in engine.generate_stream(messages, max_tokens, temperature):
+                gen_kwargs = {
+                    "max_tokens": int(max_tokens),
+                    "temperature": float(temperature),
+                    "top_p": float(top_p),
+                    "stop": stop,
+                }
+
+                for chunk in engine.generate_stream(messages, **gen_kwargs):
                     loop.call_soon_threadsafe(queue.put_nowait, chunk)
-                loop.call_soon_threadsafe(queue.put_nowait, None)
+                loop.call_soon_threadsafe(queue.put_nowait, None)  # Конец
             except Exception as e:
+                logger.error(f"Generation error: {e}")
                 loop.call_soon_threadsafe(queue.put_nowait, {"error": str(e)})
 
         loop.run_in_executor(None, worker)
 
         while True:
             chunk = await queue.get()
+
             if chunk is None:
                 yield "data: [DONE]\n\n"
                 break
 
             if isinstance(chunk, dict) and "error" in chunk:
-                yield f"data: {json.dumps({'error': chunk['error']})}\n\n"
+                err_json = json.dumps(
+                    {"error": {"message": chunk["error"], "type": "internal_error"}}
+                )
+                yield f"data: {err_json}\n\n"
                 break
 
+            # Стандартный чанк
             yield f"data: {json.dumps(chunk)}\n\n"
 
     if stream_req:
-        return StreamingResponse(stream_generator(), media_type="text/event-stream")
+        headers = {
+            "X-Accel-Buffering": "no",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "text/event-stream",
+        }
+        return StreamingResponse(
+            stream_generator(), media_type="text/event-stream", headers=headers
+        )
 
     else:
 
@@ -64,6 +108,8 @@ async def chat_completions(request: Request):
                     messages=messages,
                     max_tokens=int(max_tokens),
                     temperature=float(temperature),
+                    top_p=float(top_p),
+                    stop=stop,
                     stream=False,
                 )
 

src/core/engine.py

@@ -24,24 +24,29 @@ class ModelEngine:
                 n_ctx=settings.CONTEXT_SIZE,
                 n_threads=settings.N_THREADS,
                 n_gpu_layers=settings.N_GPU_LAYERS,
-                verbose=True,
+                verbose=False,
             )
             print("Model loaded successfully!")
         except Exception as e:
             print(f"CRITICAL ERROR loading model: {e}")
 
-    def generate_stream(
-        self, messages: List[Dict[str, str]], max_tokens: int, temperature: float
-    ) -> Generator:
+    # Изменили сигнатуру: теперь принимает **kwargs
+    def generate_stream(self, messages: List[Dict[str, str]], **kwargs) -> Generator:
         if not self.llm:
             raise RuntimeError("Model not loaded")
 
+        max_tokens = kwargs.get("max_tokens", settings.DEFAULT_MAX_TOKENS)
+        temperature = kwargs.get("temperature", settings.DEFAULT_TEMP)
+        stop = kwargs.get("stop", [])
+
         with self.lock:
             stream = self.llm.create_chat_completion(
                 messages=messages,
                 max_tokens=int(max_tokens),
                 temperature=float(temperature),
+                stop=stop,
                 stream=True,
+                top_p=kwargs.get("top_p", 0.95),
             )
             for chunk in stream:
                 yield chunk

src/ui/callbacks.py

@@ -40,7 +40,9 @@ def bot_response(history, system_prompt, temperature, max_tokens):
     history.append({"role": "assistant", "content": ""})
 
     try:
-        stream = engine.generate_stream(messages, max_tokens, temperature)
+        stream = engine.generate_stream(
+            messages=messages, max_tokens=max_tokens, temperature=temperature
+        )
 
         partial_text = ""
         for chunk in stream:

src/ui/components.py

@@ -24,7 +24,7 @@ def create_ui():
                 temp = gr.Slider(0, 1, value=settings.DEFAULT_TEMP, label="Temperature")
                 tokens = gr.Slider(
                     512,
-                    settings.DEFAULT_MAX_TOKENS,
+                    settings.CONTEXT_SIZE,
                     value=settings.DEFAULT_MAX_TOKENS,
                     label="Max New Tokens",
                     step=128,