Update app.py

app.py

@@ -3,12 +3,12 @@ from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
 from typing import List, Optional
 import uvicorn
-from ctransformers import AutoModelForCausalLM
-import torch  # For CPU checks
+from llama_cpp import Llama
+import os
 
-# Model config (Q5_K_M quantization: fast on CPU, ~300MB, high quality)
+# Model config (Official Qwen GGUF repo; Q5_K_M: fast on CPU, ~300MB, high quality)
 MODEL_REPO = "Qwen/Qwen1.5-0.5B-Chat-GGUF"
-MODEL_FILE = "qwen1_5-0_5b-chat-q5_k_m.gguf"  # Download via HF CLI if needed
+MODEL_FILE = "Qwen1.5-0.5B-Chat-Q5_K_M.gguf"  # Correct file name with dots & uppercase
 CONTEXT_LENGTH = 32768
 MAX_TOKENS = 512
 TEMPERATURE = 0.7
@@ -34,14 +34,14 @@ class ChatResponse(BaseModel):
 
 def load_model():
     global model
-    print("Loading quantized Qwen1.5-0.5B-Chat model on CPU... (10–20s)")
-    model = AutoModelForCausalLM.from_pretrained(
-        MODEL_REPO,
+    print("Loading quantized Qwen1.5-0.5B-Chat model on CPU... (10–15s)")
+    model = Llama.from_pretrained(
+        repo_id=MODEL_REPO,
         model_file=MODEL_FILE,
-        model_type="qwen",  # Qwen architecture
-        context_length=CONTEXT_LENGTH,
-        gpu_layers=0,  # Force CPU (set to >0 if GPU available)
-        threads=0  # Auto-detect CPU threads for max speed
+        n_ctx=CONTEXT_LENGTH,
+        n_threads=0,  # Auto-detect all CPU threads for max speed
+        verbose=False,  # Reduce logs
+        chat_format="chatml"  # Qwen uses ChatML template; auto-applies to messages
     )
     print("Model loaded! Ready for fast CPU inference.")
 
@@ -49,34 +49,22 @@ def load_model():
 load_model()
 
 def generate_response(messages: List[ChatMessage], max_tokens: int, temperature: float, top_p: float) -> str:
-    # Build prompt in Qwen chat format (multi-turn support)
-    prompt = ""
-    for msg in messages:
-        if msg.role == "user":
-            prompt += f"<|im_start|>user\n{msg.content}<|im_end|>\n"
-        elif msg.role == "assistant":
-            prompt += f"<|im_start|>assistant\n{msg.content}<|im_end|>\n"
-    prompt += f"<|im_start|>assistant\n"  # Start assistant response
-
-    # Generate (streams internally but we collect full output)
-    response = model(
-        prompt,
-        max_new_tokens=max_tokens,
+    # Prepare messages list (llama-cpp auto-applies Qwen chat template)
+    chat_messages = [{"role": msg.role, "content": msg.content} for msg in messages]
+    
+    # Generate using built-in chat completion (handles template, sampling, etc.)
+    response = model.create_chat_completion(
+        messages=chat_messages,
+        max_tokens=max_tokens,
         temperature=temperature,
         top_p=top_p,
-        do_sample=True,
-        stream=False  # Full response for API simplicity
+        stream=False,
+        echo=False  # Don't repeat input
     )
     
-    # Extract assistant response (strip prompt)
-    full_output = response
-    assistant_start = prompt.rfind("<|im_start|>assistant\n")
-    if assistant_start != -1:
-        response = full_output[assistant_start + len("<|im_start|>assistant\n"):].strip()
-        # Clean up any trailing <|im_end|>
-        response = response.split("<|im_end|>")[0].strip()
-    
-    return response
+    # Extract assistant response
+    bot_reply = response["choices"][0]["message"]["content"]
+    return bot_reply
 
 @app.post("/chat/", response_model=ChatResponse)
 async def chat_endpoint(request: ChatRequest):