app/utils/llm_models.py

from pathlib import Path
from langchain_community.llms import LlamaCpp
from langchain_community.chat_models import ChatLlamaCpp
from app.core.config import settings
model_file = Path(settings.model_path) / settings.local_model_name

def load_model():
    return ChatLlamaCpp(
        model_path=str(model_file),  # Direct path
        n_ctx=8192,
        n_batch=512,
        n_threads=4,
        temperature=0.05,
        top_p=0.8,
        top_k=20,
        repeat_penalty=1.1,
        f16_kv=True,
        verbose=False,
    )