refactor(chat): migrate from transformers to llama-cpp-python using Qwen 3B

app/routes/nlp.py

@@ -1,73 +1,39 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer
-from pydantic import BaseModel
 from fastapi import APIRouter, Request
-from .tts import save_audio
-import torch
+from pydantic import BaseModel
+from llama_cpp import Llama
+# from .tts import save_audio   # jeśli używasz TTS
 
-model_name = "Qwen/Qwen2.5-1.5B-Instruct"
 router = APIRouter()
 
 class ChatRequest(BaseModel):
     message: str
 
-# Load NLP model and tokenizer
+# Load NLP model
 def load_model_nlp():
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        torch_dtype=torch.float32,     # CPU-friendly
-        low_cpu_mem_usage=True          # low memory usage
+    llm = Llama.from_pretrained(
+        repo_id="Qwen/Qwen2.5-3B-Instruct-GGUF",
+        filename="qwen2.5-3b-instruct-q5_0.gguf",
+        n_ctx=2048,  
     )
-    model.to("cpu")
-    model.eval()
-    return model, tokenizer
+    return llm
 
 @router.post("/chat")
 async def chat(request: Request, message: ChatRequest):
     text = message.message
-    model = request.app.state.model_nlp
-    tokenizer = request.app.state.tokenizer_nlp
-
-    messages = [
-        {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You help users learn English."},
-        {"role": "user", "content": text},
-    ]
-
-    # apply chat template
-    text_input = tokenizer.apply_chat_template(
-        messages,
-        tokenize=False,
-        add_generation_prompt=True
-    )
-
-    # tokenize input
-    model_inputs = tokenizer([text_input], return_tensors="pt")
-
-    # generate response
-    with torch.inference_mode():
-        generated_ids = model.generate(
-            **model_inputs,
-            max_new_tokens=128,   # CPU + RAM
-            do_sample=True,      
-            temperature=0.7,      
-            top_p=0.9,            
-            top_k=50,            
-        )
+    llm = request.app.state.model_nlp
 
+    # Opcjonalnie dodaj wiadomość systemową
+    prompt = f"You are Qwen, created by Alibaba Cloud. You help users learn English.\nUser: {text}\nAssistant:"
 
-    # extract only the newly generated tokens
-    new_tokens = [
-        out_ids[len(in_ids):] for in_ids, out_ids in zip(
-            model_inputs.input_ids, generated_ids
-        )
-    ]
+    # Generowanie odpowiedzi
+    output = llm(prompt, max_tokens=128, temperature=0.7, top_p=0.9, top_k=50)
 
-    response_text = tokenizer.batch_decode(new_tokens, skip_special_tokens=True)[0]
+    response_text = output['choices'][0]['text'].strip()
 
-    # Generate audio response (commented out for testing purposes)
+    # Generate audio response (opcjonalnie)
     # url_path = save_audio(request, response_text)
 
     return {
         "response": response_text,
         "audio": 'url_path'  # placeholder
-    }
+    }