Mohd-Shujath commited on
Commit
71236ce
·
verified ·
1 Parent(s): d9a0710
Files changed (1) hide show
  1. app.py +55 -14
app.py CHANGED
@@ -1,10 +1,54 @@
1
  import os
2
  import sys
 
 
 
3
  import subprocess
4
  import re
5
 
6
  # =============================================================
7
- # 1. Bootstrap prebuilt CUDA-enabled llama-cpp-python for ZeroGPU
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  # =============================================================
9
  def bootstrap_llama_cpp():
10
  print("Bootstrapping llama-cpp-python with CUDA for ZeroGPU...")
@@ -13,7 +57,6 @@ def bootstrap_llama_cpp():
13
  api = HfApi()
14
  files = api.list_repo_files("AIencoder/llama-cpp-wheels", repo_type="dataset")
15
 
16
- # Priority 1: CUDA 12.4 / 12.6 wheels for ZeroGPU NVIDIA A10G
17
  cuda_candidates = [
18
  p for p in files
19
  if p.endswith(".whl")
@@ -21,8 +64,6 @@ def bootstrap_llama_cpp():
21
  and "basic" in p
22
  and "/0.3.36/" in p
23
  ]
24
-
25
- # Priority 2: CPU fallback
26
  cpu_candidates = [
27
  p for p in files
28
  if p.endswith(".whl")
@@ -39,13 +80,17 @@ def bootstrap_llama_cpp():
39
  subprocess.check_call([
40
  sys.executable, "-m", "pip", "install", "--force-reinstall", "--no-deps", whl_path
41
  ])
42
- print("llama-cpp-python successfully installed!")
 
 
 
43
 
44
  try:
45
  from llama_cpp import Llama
46
- print("llama-cpp-python already available.")
47
  except (ImportError, RuntimeError, OSError):
48
  bootstrap_llama_cpp()
 
49
  from llama_cpp import Llama
50
 
51
  import spaces
@@ -53,10 +98,8 @@ from typing import List
53
  import gradio as gr
54
 
55
  # =============================================================
56
- # 2. ZeroGPU Lazy Model Loader
57
  # =============================================================
58
- # In ZeroGPU, CUDA is ONLY active inside @spaces.GPU decorated functions.
59
- # Loading Llama globally at startup fails because no GPU context exists yet.
60
  MODEL_PATH = "Qwen3.5-0.8B.Q4_K_M.gguf"
61
  llm = None
62
 
@@ -65,7 +108,6 @@ def get_llm():
65
  if llm is None:
66
  print(f"Initializing {MODEL_PATH} on ZeroGPU...")
67
  try:
68
- # Full GPU layer offload to ZeroGPU NVIDIA A10G
69
  llm = Llama(
70
  model_path=MODEL_PATH,
71
  n_ctx=4096,
@@ -82,7 +124,7 @@ def get_llm():
82
  n_gpu_layers=0,
83
  verbose=False
84
  )
85
- print("Model loaded successfully in CPU mode.")
86
  return llm
87
 
88
  SYSTEM_PROMPT = (
@@ -93,11 +135,10 @@ SYSTEM_PROMPT = (
93
  )
94
 
95
  # =============================================================
96
- # 3. Inference with ZeroGPU Allocation
97
  # =============================================================
98
  @spaces.GPU(duration=30)
99
  def gradio_chat(user_message, history):
100
- # Model is loaded/invoked strictly INSIDE @spaces.GPU when CUDA context is granted
101
  model = get_llm()
102
 
103
  messages = [{"role": "system", "content": SYSTEM_PROMPT}]
@@ -121,7 +162,7 @@ def gradio_chat(user_message, history):
121
  return response["choices"][0]["message"]["content"]
122
 
123
  # =============================================================
124
- # 4. Gradio Interface (Exposing clean /chat API)
125
  # =============================================================
126
  demo = gr.ChatInterface(
127
  fn=gradio_chat,
 
1
  import os
2
  import sys
3
+ import glob
4
+ import ctypes
5
+ import site
6
  import subprocess
7
  import re
8
 
9
  # =============================================================
10
+ # 1. Ensure CUDA Runtime Libraries (libcudart.so.12) are linked
11
+ # =============================================================
12
+ def fix_cuda_libs():
13
+ """Expose bundled NVIDIA CUDA runtime libraries to dlopen for llama-cpp."""
14
+ candidate_dirs = []
15
+ for sp in site.getsitepackages():
16
+ nvidia_dir = os.path.join(sp, "nvidia")
17
+ if os.path.exists(nvidia_dir):
18
+ for root, dirs, _ in os.walk(nvidia_dir):
19
+ if "lib" in dirs:
20
+ candidate_dirs.append(os.path.join(root, "lib"))
21
+ for p in ["/usr/local/cuda/lib64", "/usr/local/cuda/lib"]:
22
+ if os.path.exists(p):
23
+ candidate_dirs.append(p)
24
+
25
+ if candidate_dirs:
26
+ current = os.environ.get("LD_LIBRARY_PATH", "")
27
+ os.environ["LD_LIBRARY_PATH"] = ":".join(candidate_dirs + ([current] if current else []))
28
+
29
+ # Preload libcudart and libcublas using RTLD_GLOBAL so libllama.so finds symbols immediately
30
+ for cdir in candidate_dirs:
31
+ for pattern in ["libcudart.so*", "libcublas.so*", "libcuda.so*"]:
32
+ for so_file in glob.glob(os.path.join(cdir, pattern)):
33
+ try:
34
+ ctypes.CDLL(so_file, mode=ctypes.RTLD_GLOBAL)
35
+ except Exception:
36
+ pass
37
+
38
+ def ensure_cuda_runtime():
39
+ fix_cuda_libs()
40
+ try:
41
+ ctypes.CDLL("libcudart.so.12", mode=ctypes.RTLD_GLOBAL)
42
+ print("libcudart.so.12 successfully loaded.")
43
+ except OSError:
44
+ print("libcudart.so.12 not found. Installing nvidia-cuda-runtime-cu12...")
45
+ subprocess.check_call([
46
+ sys.executable, "-m", "pip", "install", "nvidia-cuda-runtime-cu12", "nvidia-cublas-cu12"
47
+ ])
48
+ fix_cuda_libs()
49
+
50
+ # =============================================================
51
+ # 2. Bootstrap prebuilt CUDA-enabled llama-cpp-python for ZeroGPU
52
  # =============================================================
53
  def bootstrap_llama_cpp():
54
  print("Bootstrapping llama-cpp-python with CUDA for ZeroGPU...")
 
57
  api = HfApi()
58
  files = api.list_repo_files("AIencoder/llama-cpp-wheels", repo_type="dataset")
59
 
 
60
  cuda_candidates = [
61
  p for p in files
62
  if p.endswith(".whl")
 
64
  and "basic" in p
65
  and "/0.3.36/" in p
66
  ]
 
 
67
  cpu_candidates = [
68
  p for p in files
69
  if p.endswith(".whl")
 
80
  subprocess.check_call([
81
  sys.executable, "-m", "pip", "install", "--force-reinstall", "--no-deps", whl_path
82
  ])
83
+ print("llama-cpp-python package installed.")
84
+
85
+ # Make sure CUDA runtime is available in dynamic linker before importing llama_cpp
86
+ ensure_cuda_runtime()
87
 
88
  try:
89
  from llama_cpp import Llama
90
+ print("llama-cpp-python loaded successfully.")
91
  except (ImportError, RuntimeError, OSError):
92
  bootstrap_llama_cpp()
93
+ ensure_cuda_runtime()
94
  from llama_cpp import Llama
95
 
96
  import spaces
 
98
  import gradio as gr
99
 
100
  # =============================================================
101
+ # 3. ZeroGPU Lazy Model Loader
102
  # =============================================================
 
 
103
  MODEL_PATH = "Qwen3.5-0.8B.Q4_K_M.gguf"
104
  llm = None
105
 
 
108
  if llm is None:
109
  print(f"Initializing {MODEL_PATH} on ZeroGPU...")
110
  try:
 
111
  llm = Llama(
112
  model_path=MODEL_PATH,
113
  n_ctx=4096,
 
124
  n_gpu_layers=0,
125
  verbose=False
126
  )
127
+ print("Model loaded in CPU mode.")
128
  return llm
129
 
130
  SYSTEM_PROMPT = (
 
135
  )
136
 
137
  # =============================================================
138
+ # 4. Inference with ZeroGPU Allocation
139
  # =============================================================
140
  @spaces.GPU(duration=30)
141
  def gradio_chat(user_message, history):
 
142
  model = get_llm()
143
 
144
  messages = [{"role": "system", "content": SYSTEM_PROMPT}]
 
162
  return response["choices"][0]["message"]["content"]
163
 
164
  # =============================================================
165
+ # 5. Gradio Interface (Exposing /chat API)
166
  # =============================================================
167
  demo = gr.ChatInterface(
168
  fn=gradio_chat,