Spaces:
Sleeping
Sleeping
| from __future__ import annotations | |
| import tomllib | |
| from datetime import date | |
| from pathlib import Path | |
| import yaml | |
| def generate_config( | |
| providers_path: Path, | |
| secrets_path: Path, | |
| output_path: Path, | |
| ) -> None: | |
| providers_data = yaml.safe_load(providers_path.read_text()) | |
| secrets_doc = tomllib.loads(secrets_path.read_text()) | |
| providers = providers_data.get("providers", {}) | |
| iteration_order = providers_data.get("iteration_order", []) | |
| keys_data = secrets_doc.get("keys", {}) | |
| today = date.today() | |
| model_list: list[dict] = [] | |
| for entry in iteration_order: | |
| provider_name = entry["provider"] | |
| model_id = entry["model"] | |
| if provider_name not in providers: | |
| continue | |
| provider = providers[provider_name] | |
| api_key_ref = provider.get("api_key_ref", provider_name) | |
| if api_key_ref not in keys_data: | |
| continue | |
| secret = keys_data[api_key_ref] | |
| expires = secret.get("expires") | |
| if expires is not None: | |
| exp_date = date.fromisoformat(str(expires)) if isinstance(expires, str) else expires | |
| if exp_date < today: | |
| continue | |
| disabled_until = secret.get("disabled_until") | |
| if disabled_until is not None: | |
| dis_date = ( | |
| date.fromisoformat(str(disabled_until)) | |
| if isinstance(disabled_until, str) | |
| else disabled_until | |
| ) | |
| if dis_date > today: | |
| continue | |
| if not provider.get("supports_system_role", True): | |
| continue | |
| model_config = None | |
| for m in provider.get("models", []): | |
| if m["id"] == model_id: | |
| model_config = m | |
| break | |
| if model_config is None: | |
| continue | |
| if model_config.get("litellm_skip", False): | |
| continue | |
| if model_config.get("context_window", 0) < 50_000: | |
| continue | |
| rpm = model_config.get("rpm") | |
| if rpm is None: | |
| rpm = provider.get("rpm") | |
| tpm = model_config.get("tpm") | |
| base_url = provider["base_url"] | |
| account_id = secret.get("account_id") | |
| if account_id: | |
| base_url = base_url.replace("{cloudflare_account_id}", str(account_id)) | |
| for key_idx, api_key in enumerate(secret.get("keys", [])): | |
| litellm_params: dict = { | |
| "model": f"openai/{model_id}", | |
| "api_key": api_key, | |
| "api_base": base_url, | |
| } | |
| if rpm is not None: | |
| litellm_params["rpm"] = rpm | |
| if tpm is not None: | |
| litellm_params["tpm"] = tpm | |
| model_list.append({ | |
| "model_name": "pipeline-model", | |
| "litellm_params": litellm_params, | |
| "model_info": {"id": f"{provider_name}::{model_id}::{key_idx}"}, | |
| }) | |
| config = { | |
| "model_list": model_list, | |
| "router_settings": { | |
| "routing_strategy": "simple-shuffle", | |
| "num_retries": 3, | |
| "retry_after": 1, | |
| # Per-deployment timeout. Without this LiteLLM was sitting on a | |
| # single slow upstream for the full ~6000s aiohttp default before | |
| # giving up — that pinned a parser worker slot for ~100 minutes per | |
| # stuck call. 90s is comfortably above the p95 for parser-stage | |
| # calls (typically 30-60s on healthy providers); anything longer | |
| # is a sign the upstream is dead and we should fail over. | |
| "timeout": 90, | |
| }, | |
| "litellm_settings": { | |
| "drop_params": True, | |
| # Same intent as router_settings.timeout but applied at the | |
| # litellm.completion() layer so non-router code paths also bail. | |
| "request_timeout": 90, | |
| }, | |
| "general_settings": { | |
| "master_key": "os.environ/LITELLM_MASTER_KEY", | |
| }, | |
| } | |
| output_path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False)) | |