Defer llama_cpp import into @spaces.GPU path (ZeroGPU: CUDA libs only exist inside the GPU call)
Browse files
app.py
CHANGED
|
@@ -19,7 +19,6 @@ _load_env()
|
|
| 19 |
|
| 20 |
import gradio as gr
|
| 21 |
from huggingface_hub import hf_hub_download
|
| 22 |
-
from llama_cpp import Llama
|
| 23 |
|
| 24 |
try:
|
| 25 |
import spaces
|
|
@@ -94,6 +93,7 @@ def model_path():
|
|
| 94 |
def get_llm():
|
| 95 |
global _llm
|
| 96 |
if _llm is None:
|
|
|
|
| 97 |
_llm = Llama(
|
| 98 |
model_path=model_path(),
|
| 99 |
n_gpu_layers=N_GPU_LAYERS,
|
|
|
|
| 19 |
|
| 20 |
import gradio as gr
|
| 21 |
from huggingface_hub import hf_hub_download
|
|
|
|
| 22 |
|
| 23 |
try:
|
| 24 |
import spaces
|
|
|
|
| 93 |
def get_llm():
|
| 94 |
global _llm
|
| 95 |
if _llm is None:
|
| 96 |
+
from llama_cpp import Llama
|
| 97 |
_llm = Llama(
|
| 98 |
model_path=model_path(),
|
| 99 |
n_gpu_layers=N_GPU_LAYERS,
|