Spaces:
Sleeping
Sleeping
File size: 4,346 Bytes
71b4454 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | import logging
from typing import Dict, Any
from abc import ABC, abstractmethod
import httpx
logger = logging.getLogger("model_providers")
class BaseProvider(ABC):
def __init__(self, config: Dict[str, Any]):
self.config = config
self.api_key = config.get("api_key")
self.base_url = config.get("base_url")
self.models = config.get("models", [])
self.default_model = config.get("default_model")
@abstractmethod
async def generate(self, model: str, prompt: str, **kwargs) -> Dict[str, Any]:
pass
def get_cost(self, model: str, tokens_prompt: int, tokens_completion: int) -> float:
return (tokens_prompt + tokens_completion) * 0.000002
async def _openai_compatible_generate(
base_url: str,
api_key: str,
model: str,
prompt: str,
extra_headers: Dict[str, str] = None,
**kwargs,
) -> Dict[str, Any]:
"""
Shared implementation for any provider exposing an OpenAI-compatible
/chat/completions endpoint (Groq, Cerebras, OpenRouter all qualify).
"""
headers = {
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
}
if extra_headers:
headers.update(extra_headers)
payload = {
"model": model,
"messages": [{"role": "user", "content": prompt}],
"max_tokens": kwargs.get("max_tokens", 1024),
"temperature": kwargs.get("temperature", 0.7),
}
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.post(
f"{base_url}/chat/completions",
headers=headers,
json=payload,
)
response.raise_for_status()
data = response.json()
choice = data["choices"][0]["message"]["content"]
usage = data.get("usage", {})
return {
"text": choice,
"tokens_prompt": usage.get("prompt_tokens", 0),
"tokens_completion": usage.get("completion_tokens", 0),
"cost": 0.0,
}
class GroqClient(BaseProvider):
BASE_URL = "https://api.groq.com/openai/v1"
async def generate(self, model: str, prompt: str, **kwargs) -> Dict[str, Any]:
return await _openai_compatible_generate(
self.BASE_URL, self.api_key, model, prompt, **kwargs
)
class CerebrasClient(BaseProvider):
BASE_URL = "https://api.cerebras.ai/v1"
async def generate(self, model: str, prompt: str, **kwargs) -> Dict[str, Any]:
return await _openai_compatible_generate(
self.BASE_URL, self.api_key, model, prompt, **kwargs
)
class OpenRouterClient(BaseProvider):
BASE_URL = "https://openrouter.ai/api/v1"
async def generate(self, model: str, prompt: str, **kwargs) -> Dict[str, Any]:
return await _openai_compatible_generate(
self.BASE_URL,
self.api_key,
model,
prompt,
extra_headers={"HTTP-Referer": "https://dolor3v.studio"},
**kwargs,
)
class HuggingFaceClient(BaseProvider):
BASE_URL = "https://api-inference.huggingface.co/models"
async def generate(self, model: str, prompt: str, **kwargs) -> Dict[str, Any]:
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
payload = {
"inputs": prompt,
"parameters": {
"max_new_tokens": kwargs.get("max_tokens", 1024),
"temperature": kwargs.get("temperature", 0.7),
},
}
async with httpx.AsyncClient(timeout=60.0) as client:
response = await client.post(
f"{self.BASE_URL}/{model}",
headers=headers,
json=payload,
)
response.raise_for_status()
data = response.json()
text = ""
if isinstance(data, list) and data:
text = data[0].get("generated_text", "")
elif isinstance(data, dict):
text = data.get("generated_text", "")
return {
"text": text,
"tokens_prompt": 0,
"tokens_completion": 0,
"cost": 0.0,
}
PROVIDER_CLASSES = {
"groq": GroqClient,
"cerebras": CerebrasClient,
"openrouter": OpenRouterClient,
"huggingface": HuggingFaceClient,
}
|