handraise-dev
/

gguf-inference

Text Generation

Model card Files Files and versions

syberWolf commited on Jul 4, 2024

Commit

c676380

·

1 Parent(s): c1cb360

update device map

Files changed (1) hide show

handler.py +0 -1

handler.py CHANGED Viewed

@@ -9,7 +9,6 @@ class EndpointHandler:
         tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct", trust_remote_code=True)
         model = AutoModelForCausalLM.from_pretrained(
             "microsoft/Phi-3-mini-128k-instruct",
-            device_map="auto",
             torch_dtype=torch.bfloat16,
             device_map="cuda",
             trust_remote_code=True

         tokenizer = AutoTokenizer.from_pretrained("microsoft/Phi-3-mini-128k-instruct", trust_remote_code=True)
         model = AutoModelForCausalLM.from_pretrained(
             "microsoft/Phi-3-mini-128k-instruct",
             torch_dtype=torch.bfloat16,
             device_map="cuda",
             trust_remote_code=True