subhdotsol commited on
Commit
57c321d
·
1 Parent(s): 962dc67

feat(inference): scaffold inference.py with OpenAI client config and env vars

Browse files
Files changed (1) hide show
  1. inference.py +16 -0
inference.py ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import asyncio
3
+ import logging
4
+ from openai import OpenAI
5
+
6
+ logging.basicConfig(level=logging.INFO)
7
+ logger = logging.getLogger(__name__)
8
+
9
+ API_BASE_URL = os.environ.get("API_BASE_URL", "http://localhost:7860")
10
+ MODEL_NAME = os.environ.get("MODEL_NAME", "mistralai/Mistral-7B-Instruct-v0.3")
11
+ HF_TOKEN = os.environ.get("HF_TOKEN", "")
12
+
13
+ client = OpenAI(
14
+ base_url = f"https://api-inference.huggingface.co/models/{MODEL_NAME}/v1",
15
+ api_key = HF_TOKEN,
16
+ )