modify

data/models/llama3-1-70b.py

@@ -11,9 +11,8 @@ app = Flask(__name__)
 api_key = os.getenv("UCODE_SECRET")
 login(api_key,add_to_git_credential=True)
 
-client = InferenceClient()
-
 model_id = "meta-llama/Meta-Llama-3.1-70B-Instruct"
+client = InferenceClient(model=model_id,token=api_key)
 
 @app.route('/chat', methods=['POST'])
 @spaces.GPU(enable_queue=True)
@@ -25,23 +24,15 @@ def chat_completion():
     temperature = data[0].get('temperature', 0.7)
     top_p = data[0].get('top_p', 0.95)
 
-    chat = client.chat.completions.create(
-        model=model_id,
-        messages=user_input,
-        stream=False,
-        max_tokens=max_tokens,
-        temperature=temperature,
-        top_p=top_p
-    )
-
     try:
-        outputs = pipeline(
-            user_input,
-            max_new_tokens=max_tokens, 
+        chat = client.chat.completions.create(
+            messages=user_input,
+            stream=False,
+            max_tokens=max_tokens,
             temperature=temperature,
             top_p=top_p
         )
-        return jsonify({"status": "success", "output": outputs[0]["generated_text"][-1]})
+        return jsonify({"status": "success", "output": chat})
     except Exception as e:
         return jsonify({"status": "error", "message": str(e)})