app/models/llm.py

from __future__ import annotations

import os
import threading

from huggingface_hub import hf_hub_download
from llama_cpp import Llama

HF_REPO = os.getenv("LLAMA_HF_REPO", "ps1811/advisor-minicpm-finetuned-gguf")
HF_FILENAME = os.getenv("LLAMA_HF_FILENAME", "advisor-minicpm-q4_k_m.gguf")

_model: Llama | None = None
_init_lock = threading.Lock()


def _preload_cuda_libs() -> None:
    try:
        import ctypes

        import nvidia.cublas
        import nvidia.cuda_runtime
    except ImportError:
        return

    for module, lib_name in (
        (nvidia.cublas, "libcublas.so.12"),
        (nvidia.cuda_runtime, "libcudart.so.12"),
    ):
        lib_path = os.path.join(module.__path__[0], "lib", lib_name)
        if os.path.isfile(lib_path):
            ctypes.CDLL(lib_path, mode=ctypes.RTLD_GLOBAL)


def load_model() -> Llama:
    global _model
    print("🧠 [load_model] called", flush=True)

    if _model is not None:
        print("🧠 [load_model] returning cached model", flush=True)
        return _model

    with _init_lock:
        if _model is not None:
            return _model

        print("⬇️ [load_model] downloading model...", flush=True)
        model_path = hf_hub_download(repo_id=HF_REPO, filename=HF_FILENAME, force_download=True,)
        print(f"✅ [load_model] model downloaded at {model_path}", flush=True)

        _preload_cuda_libs()
        gpu_layers = int(os.getenv("LLAMA_GPU_LAYERS", "-1"))
        n_ctx = int(os.getenv("LLAMA_N_CTX", "2048"))
        n_threads = int(os.getenv("LLAMA_N_THREADS", "4"))
        print(
            f"🚀 [load_model] initializing Llama "
            f"(n_gpu_layers={gpu_layers}, n_ctx={n_ctx}, n_threads={n_threads})",
            flush=True,
        )

        _model = Llama(
            model_path=model_path,
            n_ctx=n_ctx,
            n_gpu_layers=gpu_layers,
            n_threads=n_threads,
            verbose=False,
        )
        print("✅ [load_model] model initialized", flush=True)

    return _model