jdesiree commited on
Commit
9be5c1b
·
verified ·
1 Parent(s): 330d0e3

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +5 -4
app.py CHANGED
@@ -315,7 +315,7 @@ class Phi3MiniEducationalLLM(Runnable):
315
  model_path,
316
  quantization_config=quant_config,
317
  device_map="auto",
318
- dtype=torch.float16,
319
  trust_remote_code=True,
320
  low_cpu_mem_usage=True,
321
  token=hf_token
@@ -343,11 +343,12 @@ class Phi3MiniEducationalLLM(Runnable):
343
  """Optimized model loading for Phi-3-mini."""
344
  self.model = AutoModelForCausalLM.from_pretrained(
345
  model_path,
346
- dtype=torch.float16, # Use float16 to save memory
347
  device_map="auto", # Let transformers decide placement
348
  trust_remote_code=True,
349
  low_cpu_mem_usage=True,
350
- token=hf_token
 
351
  )
352
 
353
  def _format_chat_template(self, prompt: str) -> str:
@@ -523,7 +524,7 @@ class Educational_Agent:
523
  start_init_and_langgraph_time = time.perf_counter()
524
  current_time = datetime.now()
525
 
526
- self.llm = Phi3MiniEducationalLLM(model_path="microsoft/Phi-3-mini-4k-instruct", use_4bit=True)
527
  self.tool_decision_engine = Tool_Decision_Engine(self.llm)
528
 
529
  # Create LangGraph workflow
 
315
  model_path,
316
  quantization_config=quant_config,
317
  device_map="auto",
318
+ torch_dtype=torch.float16,
319
  trust_remote_code=True,
320
  low_cpu_mem_usage=True,
321
  token=hf_token
 
343
  """Optimized model loading for Phi-3-mini."""
344
  self.model = AutoModelForCausalLM.from_pretrained(
345
  model_path,
346
+ torch_dtype=torch.float16, # Use float16 to save memory
347
  device_map="auto", # Let transformers decide placement
348
  trust_remote_code=True,
349
  low_cpu_mem_usage=True,
350
+ token=hf_token,
351
+ revision="0a67737cc96d2554230f90338b163bc6380a2a85" # Pin revision for security
352
  )
353
 
354
  def _format_chat_template(self, prompt: str) -> str:
 
524
  start_init_and_langgraph_time = time.perf_counter()
525
  current_time = datetime.now()
526
 
527
+ self.llm = Phi3MiniEducationalLLM(model_path="microsoft/Phi-3-mini-4k-instruct", use_4bit=False)
528
  self.tool_decision_engine = Tool_Decision_Engine(self.llm)
529
 
530
  # Create LangGraph workflow