add model

data/models/llama3-1-70b.py

@@ -0,0 +1,50 @@
+from flask import Flask, request, jsonify
+from huggingface_hub import login
+import spaces
+import transformers
+import torch
+import os
+
+# Initialize Flask app
+app = Flask(__name__)
+
+api_key = os.getenv("UCODE_SECRET")
+login(api_key,add_to_git_credential=True)
+
+model_id = "meta-llama/Meta-Llama-3.1-70B-Instruct"
+
+pipeline = transformers.pipeline(
+    "text-generation",
+    model=model_id,
+    model_kwargs={"torch_dtype": torch.bfloat16},
+    device="cuda",
+    token=True
+)
+
+@app.route('/chat', methods=['POST'])
+@spaces.GPU(enable_queue=True)
+def chat_completion():
+    data = request.json
+    
+    user_input = data[0].get('user_input', [])
+    max_tokens = data[0].get('max_tokens', 2048)
+    temperature = data[0].get('temperature', 0.7)
+    top_p = data[0].get('top_p', 0.95)
+
+    try:
+        outputs = pipeline(
+            user_input,
+            max_new_tokens=max_tokens, 
+            temperature=temperature,
+            top_p=top_p
+        )
+        return jsonify({"status": "success", "output": outputs[0]["generated_text"][-1]})
+    except Exception as e:
+        return jsonify({"status": "error", "message": str(e)})
+
+
+def main():
+    app.run(host='0.0.0.0', port=7052)
+
+if __name__ == "__main__":
+    main()