File size: 6,152 Bytes
176cf9b
56e59a7
71236ce
 
 
56e59a7
4fb13be
56e59a7
2d7b7d3
71236ce
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2d7b7d3
 
d9a0710
56e59a7
 
 
 
d9a0710
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2d7b7d3
 
d9a0710
2d7b7d3
71236ce
 
 
 
2d7b7d3
 
 
71236ce
2d7b7d3
 
71236ce
56e59a7
 
2c0a1bd
3f0f6fc
176cf9b
 
2d7b7d3
71236ce
2d7b7d3
176cf9b
d9a0710
176cf9b
d9a0710
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71236ce
d9a0710
176cf9b
 
2d7b7d3
 
 
 
176cf9b
 
2d7b7d3
71236ce
2d7b7d3
d9a0710
316f370
d9a0710
 
 
316f370
3f0f6fc
 
 
d9a0710
3f0f6fc
d9a0710
3f0f6fc
d9a0710
 
 
 
 
 
 
 
 
 
176cf9b
d9a0710
71236ce
d9a0710
316f370
 
 
d9a0710
3f0f6fc
316f370
 
5ec38d4
316f370
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
import os
import sys
import glob
import ctypes
import site
import subprocess
import re

# =============================================================
# 1. Ensure CUDA Runtime Libraries (libcudart.so.12) are linked
# =============================================================
def fix_cuda_libs():
    """Expose bundled NVIDIA CUDA runtime libraries to dlopen for llama-cpp."""
    candidate_dirs = []
    for sp in site.getsitepackages():
        nvidia_dir = os.path.join(sp, "nvidia")
        if os.path.exists(nvidia_dir):
            for root, dirs, _ in os.walk(nvidia_dir):
                if "lib" in dirs:
                    candidate_dirs.append(os.path.join(root, "lib"))
    for p in ["/usr/local/cuda/lib64", "/usr/local/cuda/lib"]:
        if os.path.exists(p):
            candidate_dirs.append(p)

    if candidate_dirs:
        current = os.environ.get("LD_LIBRARY_PATH", "")
        os.environ["LD_LIBRARY_PATH"] = ":".join(candidate_dirs + ([current] if current else []))

    # Preload libcudart and libcublas using RTLD_GLOBAL so libllama.so finds symbols immediately
    for cdir in candidate_dirs:
        for pattern in ["libcudart.so*", "libcublas.so*", "libcuda.so*"]:
            for so_file in glob.glob(os.path.join(cdir, pattern)):
                try:
                    ctypes.CDLL(so_file, mode=ctypes.RTLD_GLOBAL)
                except Exception:
                    pass

def ensure_cuda_runtime():
    fix_cuda_libs()
    try:
        ctypes.CDLL("libcudart.so.12", mode=ctypes.RTLD_GLOBAL)
        print("libcudart.so.12 successfully loaded.")
    except OSError:
        print("libcudart.so.12 not found. Installing nvidia-cuda-runtime-cu12...")
        subprocess.check_call([
            sys.executable, "-m", "pip", "install", "nvidia-cuda-runtime-cu12", "nvidia-cublas-cu12"
        ])
        fix_cuda_libs()

# =============================================================
# 2. Bootstrap prebuilt CUDA-enabled llama-cpp-python for ZeroGPU
# =============================================================
def bootstrap_llama_cpp():
    print("Bootstrapping llama-cpp-python with CUDA for ZeroGPU...")
    from huggingface_hub import HfApi, hf_hub_download

    api = HfApi()
    files = api.list_repo_files("AIencoder/llama-cpp-wheels", repo_type="dataset")

    cuda_candidates = [
        p for p in files 
        if p.endswith(".whl") 
        and ("cuda12.4" in p or "cuda12.6" in p) 
        and "basic" in p 
        and "/0.3.36/" in p
    ]
    cpu_candidates = [
        p for p in files 
        if p.endswith(".whl") 
        and "linux/x86_64/cpu/" in p 
        and "/0.3.36/" in p
    ]

    chosen = cuda_candidates[0] if cuda_candidates else (cpu_candidates[0] if cpu_candidates else None)
    if not chosen:
        raise RuntimeError("No compatible prebuilt llama-cpp-python wheel found.")

    print(f"Selected wheel for ZeroGPU: {chosen}")
    whl_path = hf_hub_download("AIencoder/llama-cpp-wheels", chosen, repo_type="dataset")
    subprocess.check_call([
        sys.executable, "-m", "pip", "install", "--force-reinstall", "--no-deps", whl_path
    ])
    print("llama-cpp-python package installed.")

# Make sure CUDA runtime is available in dynamic linker before importing llama_cpp
ensure_cuda_runtime()

try:
    from llama_cpp import Llama
    print("llama-cpp-python loaded successfully.")
except (ImportError, RuntimeError, OSError):
    bootstrap_llama_cpp()
    ensure_cuda_runtime()
    from llama_cpp import Llama

import spaces
from typing import List
import gradio as gr

# =============================================================
# 3. ZeroGPU Lazy Model Loader
# =============================================================
MODEL_PATH = "Qwen3.5-0.8B.Q4_K_M.gguf"
llm = None

def get_llm():
    global llm
    if llm is None:
        print(f"Initializing {MODEL_PATH} on ZeroGPU...")
        try:
            llm = Llama(
                model_path=MODEL_PATH,
                n_ctx=4096,
                n_gpu_layers=-1,
                verbose=False
            )
            print("Model loaded successfully with GPU offload!")
        except Exception as e:
            print(f"GPU offload warning: {e}. Falling back to CPU execution.")
            llm = Llama(
                model_path=MODEL_PATH,
                n_ctx=4096,
                n_threads=2,
                n_gpu_layers=0,
                verbose=False
            )
            print("Model loaded in CPU mode.")
    return llm

SYSTEM_PROMPT = (
    "You are Sherpa, an AI driving assistant for the CommitCon ride-hailing "
    "repositioning platform in Bengaluru. Provide concise, direct, helpful "
    "recommendations to drivers about where to reposition for high demand, "
    "earnings, and low traffic delays."
)

# =============================================================
# 4. Inference with ZeroGPU Allocation
# =============================================================
@spaces.GPU(duration=30)
def gradio_chat(user_message, history):
    model = get_llm()

    messages = [{"role": "system", "content": SYSTEM_PROMPT}]
    if history:
        for item in history:
            if isinstance(item, (list, tuple)) and len(item) == 2:
                u, b = item
                messages.append({"role": "user", "content": str(u)})
                if b:
                    messages.append({"role": "assistant", "content": str(b)})
            elif isinstance(item, dict):
                messages.append(item)

    messages.append({"role": "user", "content": user_message})

    response = model.create_chat_completion(
        messages=messages,
        temperature=0.7,
        max_tokens=256
    )
    return response["choices"][0]["message"]["content"]

# =============================================================
# 5. Gradio Interface (Exposing /chat API)
# =============================================================
demo = gr.ChatInterface(
    fn=gradio_chat,
    title="🧭 CommitCon Sherpa Driver Copilot",
    description="Fine-tuned Qwen 0.8B Q4_K_M on ZeroGPU — llama-cpp-python",
    api_name="chat"
)

if __name__ == "__main__":
    demo.launch()