debug

data/models/llama3-1-70b.py

@@ -14,7 +14,7 @@ login(api_key,add_to_git_credential=True)
 
 model_id = "hugging-quants/Meta-Llama-3.1-70B-Instruct-AWQ-INT4"
 
-tokenizer   = AutoTokenizer.from_pretrained(model_id)
+tokenizer   = AutoTokenizer.from_pretrained(model_id,token=True)
 model       = AutoModelForCausalLM.from_pretrained(
                 model_id,
                 torch_dtype=torch.float16,
@@ -22,7 +22,7 @@ model       = AutoModelForCausalLM.from_pretrained(
                 device_map="sequential",
                 offload_folder="offload",
                 offload_state_dict=True,
-                use_auth_token=True
+                token=True
             )
 
 @app.route('/chat', methods=['POST'])
@@ -36,7 +36,7 @@ def chat_completion():
     top_p = data[0].get('top_p', 0.95)
 
     try:
-        input_ids = tokenizer.apply_chat_template(user_input, tokenize=False, add_generation_prompt=True)
+        input_ids = tokenizer.apply_chat_template(user_input, tokenize=False, add_generation_prompt=True,token=True)
         inputs = tokenizer(
                     input_ids,
                     return_tensors="pt",