Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -315,7 +315,7 @@ class Phi3MiniEducationalLLM(Runnable):
|
|
| 315 |
model_path,
|
| 316 |
quantization_config=quant_config,
|
| 317 |
device_map="auto",
|
| 318 |
-
|
| 319 |
trust_remote_code=True,
|
| 320 |
low_cpu_mem_usage=True,
|
| 321 |
token=hf_token
|
|
@@ -343,11 +343,12 @@ class Phi3MiniEducationalLLM(Runnable):
|
|
| 343 |
"""Optimized model loading for Phi-3-mini."""
|
| 344 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 345 |
model_path,
|
| 346 |
-
|
| 347 |
device_map="auto", # Let transformers decide placement
|
| 348 |
trust_remote_code=True,
|
| 349 |
low_cpu_mem_usage=True,
|
| 350 |
-
token=hf_token
|
|
|
|
| 351 |
)
|
| 352 |
|
| 353 |
def _format_chat_template(self, prompt: str) -> str:
|
|
@@ -523,7 +524,7 @@ class Educational_Agent:
|
|
| 523 |
start_init_and_langgraph_time = time.perf_counter()
|
| 524 |
current_time = datetime.now()
|
| 525 |
|
| 526 |
-
self.llm = Phi3MiniEducationalLLM(model_path="microsoft/Phi-3-mini-4k-instruct", use_4bit=
|
| 527 |
self.tool_decision_engine = Tool_Decision_Engine(self.llm)
|
| 528 |
|
| 529 |
# Create LangGraph workflow
|
|
|
|
| 315 |
model_path,
|
| 316 |
quantization_config=quant_config,
|
| 317 |
device_map="auto",
|
| 318 |
+
torch_dtype=torch.float16,
|
| 319 |
trust_remote_code=True,
|
| 320 |
low_cpu_mem_usage=True,
|
| 321 |
token=hf_token
|
|
|
|
| 343 |
"""Optimized model loading for Phi-3-mini."""
|
| 344 |
self.model = AutoModelForCausalLM.from_pretrained(
|
| 345 |
model_path,
|
| 346 |
+
torch_dtype=torch.float16, # Use float16 to save memory
|
| 347 |
device_map="auto", # Let transformers decide placement
|
| 348 |
trust_remote_code=True,
|
| 349 |
low_cpu_mem_usage=True,
|
| 350 |
+
token=hf_token,
|
| 351 |
+
revision="0a67737cc96d2554230f90338b163bc6380a2a85" # Pin revision for security
|
| 352 |
)
|
| 353 |
|
| 354 |
def _format_chat_template(self, prompt: str) -> str:
|
|
|
|
| 524 |
start_init_and_langgraph_time = time.perf_counter()
|
| 525 |
current_time = datetime.now()
|
| 526 |
|
| 527 |
+
self.llm = Phi3MiniEducationalLLM(model_path="microsoft/Phi-3-mini-4k-instruct", use_4bit=False)
|
| 528 |
self.tool_decision_engine = Tool_Decision_Engine(self.llm)
|
| 529 |
|
| 530 |
# Create LangGraph workflow
|