| import os |
| from openai import AsyncOpenAI |
| from dotenv import load_dotenv |
|
|
| load_dotenv() |
|
|
| PROVIDER_CONFIG = { |
| "local": { |
| "base_url": "http://localhost:11434/v1", |
| "api_key": "OLLAMA_API_KEY", |
| "models": [ |
| "qwen2.5:7b", |
| "qwen2.5:3b", |
| "llama3.1:8b", |
| ], |
| }, |
| "modal": { |
| "base_url": os.getenv("MODAL_BASE_URL"), |
| "api_key": "MODAL_API_KEY", |
| "models": [ |
| "google/gemma-4-26B-A4B-it", |
| ], |
| }, |
| "nvidia": { |
| "base_url": "https://integrate.api.nvidia.com/v1", |
| "api_key": "NVIDIA_NIM_API_KEY", |
| "models": [ |
| "minimaxai/minimax-m2.7", |
| "stepfun-ai/step-3.5-flash", |
| "z-ai/glm4.7", |
| "deepseek-ai/deepseek-v3.2", |
| "moonshotai/kimi-k2-thinking", |
| "mistralai/devstral-2-123b-instruct-2512", |
| "mistralai/mistral-large-3-675b-instruct-2512", |
| "bytedance/seed-oss-36b-instruct", |
| "qwen/qwen3-coder-480b-a35b-instruct", |
| "nvidia/nemotron-3-ultra-550b-a55b", |
| ], |
| }, |
| "zai": { |
| "base_url": "https://api.z.ai/api/paas/v4/", |
| "api_key": "ZAI_API_KEY", |
| "models": [ |
| "glm-4.7-flash", |
| "glm-4.6v-flash", |
| ], |
| }, |
| } |
|
|
| DEFAULT_PROVIDER = os.getenv("LLM_PROVIDER", "modal") |
|
|
|
|
| def get_llm_client(provider: str | None = None) -> AsyncOpenAI: |
| provider = provider or DEFAULT_PROVIDER |
| cfg = PROVIDER_CONFIG.get(provider) or PROVIDER_CONFIG[DEFAULT_PROVIDER] |
| api_key = os.getenv(cfg["api_key"]) or "ollama" |
| return AsyncOpenAI(base_url=cfg["base_url"], api_key=api_key) |
|
|
