Update app.py

app.py

@@ -1,27 +1,37 @@
 import json
+import traceback
 
 import gradio as gr
 import uvicorn
 from fastapi import FastAPI, Request
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi.responses import JSONResponse, RedirectResponse, StreamingResponse
+from fastapi.responses import JSONResponse, StreamingResponse
 from gradio import mount_gradio_app
 from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
 
+# Конфигурация модели
 REPO_ID = "Qwen/Qwen2.5-Coder-7B-Instruct-GGUF"
 FILENAME = "qwen2.5-coder-7b-instruct-q5_k_m.gguf"
 
+CONTEXT_SIZE = 8192
+MAX_OUTPUT_TOKENS = 4096
+
 print(f"Loading model {REPO_ID}...")
-model_path = hf_hub_download(repo_id=REPO_ID, filename=FILENAME)
-
-print("Initializing Llama...")
-llm = Llama(
-    model_path=model_path,
-    n_ctx=8192,
-    n_threads=2,
-    verbose=True,
-)
+try:
+    model_path = hf_hub_download(repo_id=REPO_ID, filename=FILENAME)
+
+    print("Initializing Llama...")
+    llm = Llama(
+        model_path=model_path,
+        n_ctx=CONTEXT_SIZE,
+        n_threads=2,
+        n_batch=512,
+        verbose=True,
+    )
+except Exception as e:
+    print(f"Critical Error loading model: {e}")
+    raise e
 
 app = FastAPI()
 
@@ -36,66 +46,105 @@ app.add_middleware(
 
 @app.post("/v1/chat/completions")
 async def chat_completions(request: Request):
-    data = await request.json()
-    messages = data.get("messages", [])
-    stream = data.get("stream", False)
-    temperature = data.get("temperature", 0.2)
-    max_tokens = data.get("max_tokens", 2048)
-
-    if not messages:
-        return JSONResponse(content={"error": "No messages provided"}, status_code=400)
-
-    output = llm.create_chat_completion(
-        messages=messages, max_tokens=max_tokens, temperature=temperature, stream=stream
-    )
-
-    if stream:
-
-        def iter_content():
-            try:
-                for chunk in output:
-                    yield f"data: {json.dumps(chunk)}\n\n"
-            except Exception as e:
-                print(f"Streaming error: {e}")
-            finally:
-                yield "data: [DONE]\n\n"
-
-        return StreamingResponse(
-            iter_content(),
-            media_type="text/event-stream",
-            headers={"Cache-Control": "no-cache", "Connection": "keep-alive"},
+    try:
+        data = await request.json()
+        messages = data.get("messages", [])
+        stream = data.get("stream", False)
+        temperature = data.get("temperature", 0.4)
+        max_tokens = data.get("max_tokens", MAX_OUTPUT_TOKENS)
+
+        if not messages:
+            return JSONResponse(
+                content={"error": "No messages provided"}, status_code=400
+            )
+
+        output = llm.create_chat_completion(
+            messages=messages,
+            max_tokens=max_tokens,
+            temperature=temperature,
+            stream=stream,
         )
 
-    return JSONResponse(content=output)
+        if stream:
+
+            def iter_content():
+                try:
+                    for chunk in output:
+                        yield f"data: {json.dumps(chunk)}\n\n"
+                except Exception as e:
+                    print(f"Streaming error: {e}")
+                    err_chunk = {
+                        "choices": [
+                            {
+                                "delta": {"content": f"\n[ERROR]: {str(e)}"},
+                                "finish_reason": "error",
+                            }
+                        ]
+                    }
+                    yield f"data: {json.dumps(err_chunk)}\n\n"
+                finally:
+                    yield "data: [DONE]\n\n"
+
+            return StreamingResponse(
+                iter_content(),
+                media_type="text/event-stream",
+                headers={"Cache-Control": "no-cache", "Connection": "keep-alive"},
+            )
+
+        return JSONResponse(content=output)
+
+    except Exception as e:
+        print(f"API Error: {e}")
+        return JSONResponse(content={"error": str(e)}, status_code=500)
 
 
 def gradio_interface(message, history):
-    messages = [{"role": "system", "content": "You are an expert coding assistant."}]
-    for u, a in history:
+    messages = [
+        {
+            "role": "system",
+            "content": "You are an expert coding assistant. Write clean, efficient code.",
+        }
+    ]
+
+    history_subset = history[-10:] if len(history) > 10 else history
+
+    for u, a in history_subset:
         messages.append({"role": "user", "content": u})
         messages.append({"role": "assistant", "content": a})
     messages.append({"role": "user", "content": message})
 
-    response_stream = llm.create_chat_completion(
-        messages=messages, max_tokens=2048, temperature=0.4, stream=True
-    )
-
     partial_text = ""
-    for chunk in response_stream:
-        delta = chunk["choices"][0]["delta"]
-        if "content" in delta:
-            partial_text += delta["content"]
-            yield partial_text
+    try:
+        response_stream = llm.create_chat_completion(
+            messages=messages,
+            max_tokens=MAX_OUTPUT_TOKENS,
+            temperature=0.4,
+            stream=True,
+        )
+
+        for chunk in response_stream:
+            delta = chunk["choices"][0]["delta"]
+            if "content" in delta:
+                partial_text += delta["content"]
+                yield partial_text
+
+    except Exception as e:
+        traceback.print_exc()
+        error_msg = f"\n\n🚫 **Error:** {str(e)}\nTry refreshing the page or shortening the context."
+        yield partial_text + error_msg
 
 
 demo = gr.ChatInterface(
     fn=gradio_interface,
-    title="Qwen 2.5 Coder API",
-    description="API endpoint: /v1/chat/completions",
+    title="Qwen 2.5 Coder (7B-Instruct)",
+    description="Running on CPU. Generation might be slow. Please be patient.",
+    examples=[
+        "Write a Python script to scrape a website.",
+        "Explain how asyncio works in Python.",
+    ],
 )
 
 app = mount_gradio_app(app, demo, path="/")
 
-
 if __name__ == "__main__":
     uvicorn.run(app, host="0.0.0.0", port=7860)