fix: load LoRA adapter weights to CPU for ZeroGPU startup compat
Browse files- model_inference.py +5 -1
model_inference.py
CHANGED
|
@@ -88,7 +88,11 @@ class TransformersLLM:
|
|
| 88 |
|
| 89 |
if adapter:
|
| 90 |
print(f"🔗 Applying LoRA adapter: {adapter}")
|
| 91 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
|
| 93 |
model.eval()
|
| 94 |
self.model = model
|
|
|
|
| 88 |
|
| 89 |
if adapter:
|
| 90 |
print(f"🔗 Applying LoRA adapter: {adapter}")
|
| 91 |
+
# torch_device="cpu": load adapter weights to CPU first. On
|
| 92 |
+
# ZeroGPU, safetensors loading straight to cuda fails at startup
|
| 93 |
+
# ("No CUDA GPUs are available") — copying CPU tensors into the
|
| 94 |
+
# model's (emulated) CUDA params works fine.
|
| 95 |
+
model = PeftModel.from_pretrained(model, adapter, torch_device="cpu")
|
| 96 |
|
| 97 |
model.eval()
|
| 98 |
self.model = model
|