arch restruct

src/api/routes.py

@@ -1,8 +1,9 @@
 import asyncio
 import json
+import threading
 
-from fastapi import APIRouter, Request
-from fastapi.responses import JSONResponse, StreamingResponse
+from fastapi import APIRouter, HTTPException, Request
+from fastapi.responses import StreamingResponse
 
 from src.core.config import settings
 from src.core.engine import engine
@@ -14,105 +15,57 @@ router = APIRouter()
 @router.post("/chat/completions")
 async def chat_completions(request: Request):
     if not engine.llm:
-        return JSONResponse({"error": "Model not loaded"}, status_code=500)
+        raise HTTPException(status_code=500, detail="Model not loaded")
 
     data = await request.json()
     messages = [
         {"role": m.get("role", "user"), "content": get_clean_text(m.get("content"))}
         for m in data.get("messages", [])
     ]
+    max_tokens = data.get("max_tokens", settings.DEFAULT_MAX_TOKENS)
+    temperature = data.get("temperature", settings.DEFAULT_TEMP)
+    stream_req = data.get("stream", True)
 
-    stream = data.get("stream", True)
-
+    # --- Логика Streaming ---
     async def stream_generator():
-        # Ensure sequential processing: acquire the engine lock for the whole generation
-        async with engine.lock:
-            import threading
-
-            # Use an asyncio.Queue to safely transfer chunks from a blocking worker thread to the async generator
-            q: asyncio.Queue = asyncio.Queue()
-            stop_event = threading.Event()
-            loop = asyncio.get_running_loop()
-
-            def worker():
-                try:
-                    for chunk in engine.llm.create_chat_completion(
-                        messages=messages,
-                        max_tokens=int(
-                            data.get("max_tokens", settings.DEFAULT_MAX_TOKENS)
-                        ),
-                        temperature=float(
-                            data.get("temperature", settings.DEFAULT_TEMP)
-                        ),
-                        stream=True,
-                    ):
-                        # stop early if requested (e.g. client disconnected)
-                        if stop_event.is_set():
-                            break
-                        # chunk_count += 1
-                        # now = time.time()
-                        loop.call_soon_threadsafe(q.put_nowait, chunk)
-                    #     loop.call_soon_threadsafe(
-                    #         q.put_nowait,
-                    #         {"__chunk": chunk, "ts": now, "count": chunk_count},
-                    #     )
-                    # # финальный лог
-                    # loop.call_soon_threadsafe(
-                    #     q.put_nowait,
-                    #     {
-                    #         "__done": True,
-                    #         "duration": time.time() - start,
-                    #         "chunks": chunk_count,
-                    #     },
-                    # )
-                except Exception as e:
-                    # Pass exception to the async side so we can surface an error or terminate cleanly
-                    loop.call_soon_threadsafe(q.put_nowait, {"__error": str(e)})
-                finally:
-                    # Sentinel to mark completion
-                    loop.call_soon_threadsafe(q.put_nowait, None)
-
-            # Run the blocking model iteration in a thread so it doesn't block the event loop
-            worker_future = loop.run_in_executor(None, worker)
+        queue = asyncio.Queue()
+        loop = asyncio.get_running_loop()
 
+        def worker():
             try:
-                while True:
-                    item = await q.get()
-                    if item is None:
-                        # worker finished normally
-                        break
-                    # If worker reported an error, stream it and break
-                    if isinstance(item, dict) and item.get("__error"):
-                        yield f"data: {json.dumps({'error': item['__error']})}\n\n"
-                        break
-                    yield f"data: {json.dumps(item)}\n\n"
+                for chunk in engine.generate_stream(messages, max_tokens, temperature):
+                    loop.call_soon_threadsafe(queue.put_nowait, chunk)
+                loop.call_soon_threadsafe(queue.put_nowait, None)
+            except Exception as e:
+                loop.call_soon_threadsafe(queue.put_nowait, {"error": str(e)})
+
+        loop.run_in_executor(None, worker)
+
+        while True:
+            chunk = await queue.get()
+            if chunk is None:
                 yield "data: [DONE]\n\n"
-            except asyncio.CancelledError:
-                # Client disconnected: signal the worker to stop and wait for it to finish, then re-raise to terminate streaming
-                stop_event.set()
-                try:
-                    await worker_future
-                except Exception:
-                    pass
-                raise
-            finally:
-                # Ensure worker is signalled to stop and awaited (idempotent)
-                stop_event.set()
-                try:
-                    await worker_future
-                except Exception:
-                    pass
-
-    if stream:
+                break
+
+            if isinstance(chunk, dict) and "error" in chunk:
+                yield f"data: {json.dumps({'error': chunk['error']})}\n\n"
+                break
+
+            yield f"data: {json.dumps(chunk)}\n\n"
+
+    if stream_req:
         return StreamingResponse(stream_generator(), media_type="text/event-stream")
 
     else:
-        async with engine.lock:
-            result = await asyncio.to_thread(
-                engine.generate,
-                messages,
-                data.get("max_tokens", settings.DEFAULT_MAX_TOKENS),
-                data.get("temperature", settings.DEFAULT_TEMP),
-                stream=False,
-            )
-            return result
+
+        def run_sync():
+            with engine.lock:
+                return engine.llm.create_chat_completion(
+                    messages=messages,
+                    max_tokens=int(max_tokens),
+                    temperature=float(temperature),
+                    stream=False,
+                )
+
+        response = await asyncio.to_thread(run_sync)
+        return response

src/core/config.py

@@ -7,10 +7,10 @@ class Settings(BaseSettings):
     REPO_ID: str = "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF"
     FILENAME: str = "qwen2.5-coder-7b-instruct-q5_k_m.gguf"
 
-    CONTEXT_SIZE: int = 8192
-    DEFAULT_MAX_TOKENS: int = 4096
+    CONTEXT_SIZE: int = 65536
+    DEFAULT_MAX_TOKENS: int = 16384
     DEFAULT_TEMP: float = 0.4
-    N_THREADS: int = os.cpu_count()
+    N_THREADS: int = 2
     N_GPU_LAYERS: int = 0
 
 

src/core/engine.py

@@ -1,4 +1,5 @@
-import asyncio
+import threading
+from typing import Any, Dict, Generator, List
 
 from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
@@ -9,11 +10,12 @@ from src.core.config import settings
 class ModelEngine:
     def __init__(self):
         self.llm = None
-        self.lock = asyncio.Lock()
+        self.lock = threading.Lock()
         self._load_model()
 
     def _load_model(self):
         try:
+            print(f"Downloading/Loading model: {settings.REPO_ID}...")
             model_path = hf_hub_download(
                 repo_id=settings.REPO_ID, filename=settings.FILENAME
             )
@@ -24,19 +26,25 @@ class ModelEngine:
                 n_gpu_layers=settings.N_GPU_LAYERS,
                 verbose=True,
             )
+            print("Model loaded successfully!")
         except Exception as e:
-            print(f"Error loading model: {e}")
+            print(f"CRITICAL ERROR loading model: {e}")
 
-    def generate(self, messages, max_tokens, temperature, stream=True):
+    def generate_stream(
+        self, messages: List[Dict[str, str]], max_tokens: int, temperature: float
+    ) -> Generator:
         if not self.llm:
             raise RuntimeError("Model not loaded")
-        return self.llm.create_chat_completion(
-            messages=messages,
-            max_tokens=int(max_tokens),
-            temperature=float(temperature),
-            stream=stream,
-        )
+
+        with self.lock:
+            stream = self.llm.create_chat_completion(
+                messages=messages,
+                max_tokens=int(max_tokens),
+                temperature=float(temperature),
+                stream=True,
+            )
+            for chunk in stream:
+                yield chunk
 
 
-# Создаем синглтон
 engine = ModelEngine()

src/ui/callbacks.py

@@ -1,20 +1,12 @@
 import gradio as gr
-from src.utils.helpers import get_clean_text
 
 from src.core.engine import engine
-
-
-def user_input(user_message, history):
-    if not user_message:
-        return None, history
-    history = history or []
-    history.append({"role": "user", "content": str(user_message)})
-    return "", history
+from src.utils.helpers import get_clean_text
 
 
 def bot_response(history, system_prompt, temperature, max_tokens):
     messages = [{"role": "system", "content": system_prompt}]
-    for msg in history[-15:]:
+    for msg in history[-7:]:
         messages.append(
             {"role": msg["role"], "content": get_clean_text(msg["content"])}
         )
@@ -22,7 +14,8 @@ def bot_response(history, system_prompt, temperature, max_tokens):
     history.append({"role": "assistant", "content": ""})
 
     try:
-        stream = engine.generate(messages, max_tokens, temperature, stream=True)
+        stream = engine.generate_stream(messages, max_tokens, temperature)
+
         partial_text = ""
         for chunk in stream:
             delta = chunk["choices"][0]["delta"]
@@ -30,16 +23,7 @@ def bot_response(history, system_prompt, temperature, max_tokens):
                 partial_text += delta["content"]
                 history[-1]["content"] = partial_text
                 yield history
+
     except Exception as e:
         history[-1]["content"] += f"\n\n❌ Error: {str(e)}"
         yield history
-
-
-def set_interactive(is_interactive):
-    return (
-        gr.update(
-            interactive=is_interactive,
-            placeholder="Wait..." if not is_interactive else "Type...",
-        ),
-        gr.update(interactive=is_interactive),
-    )

src/ui/components.py

@@ -1,11 +1,5 @@
 import gradio as gr
 
-# if not hasattr(gr, "Separator"):
-#     def _gr_separator():
-#         return gr.HTML(
-#             "<div style='margin: 15px 0; border-top: 1px solid var(--border-color-primary);'></div>"
-#         )
-#     gr.Separator = lambda *args, **kwargs: _gr_separator()
 from src.core.config import settings
 from src.ui.callbacks import bot_response, set_interactive, user_input
 from src.ui.styles import CSS
@@ -24,7 +18,7 @@ def create_ui():
                 gr.Markdown("### ⚙️ Model Settings")
                 sys_pt = gr.Textbox(
                     label="System Prompt",
-                    value="Вы опытный программист. Отвечаете кратко и по делу.",
+                    value="Вы опытный программист. Отвечаете кратко и по делу, пишите качественный и рабочий код.",
                     lines=4,
                 )
                 temp = gr.Slider(0, 1, value=settings.DEFAULT_TEMP, label="Temperature")