jdesiree commited on
Commit
324c8ba
·
verified ·
1 Parent(s): 293ae98

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +3 -3
app.py CHANGED
@@ -301,7 +301,7 @@ class Phi2EducationalLLM(Runnable):
301
  model_path,
302
  quantization_config=quant_config,
303
  device_map="auto",
304
- torch_dtype=torch.float16,
305
  trust_remote_code=True,
306
  low_cpu_mem_usage=True
307
  )
@@ -328,7 +328,7 @@ class Phi2EducationalLLM(Runnable):
328
  """Optimized model loading for 16GB RAM systems."""
329
  self.model = AutoModelForCausalLM.from_pretrained(
330
  model_path,
331
- torch_dtype=torch.float16, # Use float16 to save memory
332
  device_map="cpu", # Force CPU for stability
333
  trust_remote_code=True,
334
  low_cpu_mem_usage=True,
@@ -345,7 +345,7 @@ class Phi2EducationalLLM(Runnable):
345
 
346
  self.model = AutoModelForCausalLM.from_pretrained(
347
  fallback_model,
348
- torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
349
  device_map="cpu",
350
  trust_remote_code=True,
351
  low_cpu_mem_usage=True
 
301
  model_path,
302
  quantization_config=quant_config,
303
  device_map="auto",
304
+ dtype=torch.float16,
305
  trust_remote_code=True,
306
  low_cpu_mem_usage=True
307
  )
 
328
  """Optimized model loading for 16GB RAM systems."""
329
  self.model = AutoModelForCausalLM.from_pretrained(
330
  model_path,
331
+ dtype=torch.float16, # Use float16 to save memory
332
  device_map="cpu", # Force CPU for stability
333
  trust_remote_code=True,
334
  low_cpu_mem_usage=True,
 
345
 
346
  self.model = AutoModelForCausalLM.from_pretrained(
347
  fallback_model,
348
+ dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
349
  device_map="cpu",
350
  trust_remote_code=True,
351
  low_cpu_mem_usage=True