from __future__ import annotations import tomllib from datetime import date from pathlib import Path import yaml def generate_config( providers_path: Path, secrets_path: Path, output_path: Path, ) -> None: providers_data = yaml.safe_load(providers_path.read_text()) secrets_doc = tomllib.loads(secrets_path.read_text()) providers = providers_data.get("providers", {}) iteration_order = providers_data.get("iteration_order", []) keys_data = secrets_doc.get("keys", {}) today = date.today() model_list: list[dict] = [] for entry in iteration_order: provider_name = entry["provider"] model_id = entry["model"] if provider_name not in providers: continue provider = providers[provider_name] api_key_ref = provider.get("api_key_ref", provider_name) if api_key_ref not in keys_data: continue secret = keys_data[api_key_ref] expires = secret.get("expires") if expires is not None: exp_date = date.fromisoformat(str(expires)) if isinstance(expires, str) else expires if exp_date < today: continue disabled_until = secret.get("disabled_until") if disabled_until is not None: dis_date = ( date.fromisoformat(str(disabled_until)) if isinstance(disabled_until, str) else disabled_until ) if dis_date > today: continue if not provider.get("supports_system_role", True): continue model_config = None for m in provider.get("models", []): if m["id"] == model_id: model_config = m break if model_config is None: continue if model_config.get("litellm_skip", False): continue if model_config.get("context_window", 0) < 50_000: continue rpm = model_config.get("rpm") if rpm is None: rpm = provider.get("rpm") tpm = model_config.get("tpm") base_url = provider["base_url"] account_id = secret.get("account_id") if account_id: base_url = base_url.replace("{cloudflare_account_id}", str(account_id)) for key_idx, api_key in enumerate(secret.get("keys", [])): litellm_params: dict = { "model": f"openai/{model_id}", "api_key": api_key, "api_base": base_url, } if rpm is not None: litellm_params["rpm"] = rpm if tpm is not None: litellm_params["tpm"] = tpm model_list.append({ "model_name": "pipeline-model", "litellm_params": litellm_params, "model_info": {"id": f"{provider_name}::{model_id}::{key_idx}"}, }) config = { "model_list": model_list, "router_settings": { "routing_strategy": "simple-shuffle", "num_retries": 3, "retry_after": 1, # Per-deployment timeout. Without this LiteLLM was sitting on a # single slow upstream for the full ~6000s aiohttp default before # giving up — that pinned a parser worker slot for ~100 minutes per # stuck call. 90s is comfortably above the p95 for parser-stage # calls (typically 30-60s on healthy providers); anything longer # is a sign the upstream is dead and we should fail over. "timeout": 90, }, "litellm_settings": { "drop_params": True, # Same intent as router_settings.timeout but applied at the # litellm.completion() layer so non-router code paths also bail. "request_timeout": 90, }, "general_settings": { "master_key": "os.environ/LITELLM_MASTER_KEY", }, } output_path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False))