Kasualdad commited on
Commit
e8c46ef
·
1 Parent(s): de794a7

fix: load LoRA adapter weights to CPU for ZeroGPU startup compat

Browse files
Files changed (1) hide show
  1. model_inference.py +5 -1
model_inference.py CHANGED
@@ -88,7 +88,11 @@ class TransformersLLM:
88
 
89
  if adapter:
90
  print(f"🔗 Applying LoRA adapter: {adapter}")
91
- model = PeftModel.from_pretrained(model, adapter)
 
 
 
 
92
 
93
  model.eval()
94
  self.model = model
 
88
 
89
  if adapter:
90
  print(f"🔗 Applying LoRA adapter: {adapter}")
91
+ # torch_device="cpu": load adapter weights to CPU first. On
92
+ # ZeroGPU, safetensors loading straight to cuda fails at startup
93
+ # ("No CUDA GPUs are available") — copying CPU tensors into the
94
+ # model's (emulated) CUDA params works fine.
95
+ model = PeftModel.from_pretrained(model, adapter, torch_device="cpu")
96
 
97
  model.eval()
98
  self.model = model