CrazyMonkey0 commited on
Commit
5d51d0f
·
1 Parent(s): 9ea2744

feat(nlp): switch Qwen2.5 model to 1.5B GGUF q6_k version

Browse files
Files changed (1) hide show
  1. app/routes/nlp.py +3 -5
app/routes/nlp.py CHANGED
@@ -24,13 +24,11 @@ class ChatRequest(BaseModel):
24
 
25
  # Load NLP model
26
  def load_model_nlp():
 
27
  llm = Llama.from_pretrained(
28
- repo_id="Qwen/Qwen2.5-3B-Instruct-GGUF",
29
- filename="qwen2.5-3b-instruct-q5_0.gguf",
30
- n_ctx=2048,
31
- verbose=False
32
  )
33
- print("[INFO] NLP model loaded.")
34
  return llm
35
 
36
  @router.post("/chat")
 
24
 
25
  # Load NLP model
26
  def load_model_nlp():
27
+
28
  llm = Llama.from_pretrained(
29
+ repo_id="Qwen/Qwen2.5-1.5B-Instruct-GGUF",
30
+ filename="qwen2.5-1.5b-instruct-q6_k.gguf",
 
 
31
  )
 
32
  return llm
33
 
34
  @router.post("/chat")