Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -301,7 +301,7 @@ class Phi2EducationalLLM(Runnable):
|
|
| 301 |
model_path,
|
| 302 |
quantization_config=quant_config,
|
| 303 |
device_map="auto",
|
| 304 |
-
|
| 305 |
trust_remote_code=True,
|
| 306 |
low_cpu_mem_usage=True
|
| 307 |
)
|
|
@@ -328,7 +328,7 @@ class Phi2EducationalLLM(Runnable):
|
|
| 328 |
"""Optimized model loading for 16GB RAM systems."""
|
| 329 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 330 |
model_path,
|
| 331 |
-
|
| 332 |
device_map="cpu", # Force CPU for stability
|
| 333 |
trust_remote_code=True,
|
| 334 |
low_cpu_mem_usage=True,
|
|
@@ -345,7 +345,7 @@ class Phi2EducationalLLM(Runnable):
|
|
| 345 |
|
| 346 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 347 |
fallback_model,
|
| 348 |
-
|
| 349 |
device_map="cpu",
|
| 350 |
trust_remote_code=True,
|
| 351 |
low_cpu_mem_usage=True
|
|
|
|
| 301 |
model_path,
|
| 302 |
quantization_config=quant_config,
|
| 303 |
device_map="auto",
|
| 304 |
+
dtype=torch.float16,
|
| 305 |
trust_remote_code=True,
|
| 306 |
low_cpu_mem_usage=True
|
| 307 |
)
|
|
|
|
| 328 |
"""Optimized model loading for 16GB RAM systems."""
|
| 329 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 330 |
model_path,
|
| 331 |
+
dtype=torch.float16, # Use float16 to save memory
|
| 332 |
device_map="cpu", # Force CPU for stability
|
| 333 |
trust_remote_code=True,
|
| 334 |
low_cpu_mem_usage=True,
|
|
|
|
| 345 |
|
| 346 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 347 |
fallback_model,
|
| 348 |
+
dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
|
| 349 |
device_map="cpu",
|
| 350 |
trust_remote_code=True,
|
| 351 |
low_cpu_mem_usage=True
|