glm router: encrypted multi-harness proxy (683 keys)

KEY FIX: GLM returns drained-key errors (code 1113) with HTTP 429 status,
not real rate-limits. The proxy was short-banning (20s) drained keys so
every request re-walked ~640 dead keys. Now body is inspected: 1113 ->
1h ban, genuine 429 -> 20s. glm-4.5-air now 0.5s, glm-4.6 ~3-5s (its
own reasoning floor).

Shared connection pool, bootstrap classify-all on startup + recovery-only
sweeps, tiered ban TTLs, multi-harness (OpenAI+Anthropic+Gemini paths),
round-robin + per-key retry, encrypted models.json (AES_KEY), instant
/health, / and /v1/models public, docs blocked.

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,8 @@
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+models.json
+*.tmp
+.env
+test_*.py

Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.11-slim
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+COPY app.py models.json.enc ./
+EXPOSE 7860
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,9 @@
+---
+title: glm
+emoji: 🐝
+colorFrom: blue
+colorTo: green
+sdk: docker
+app_port: 7860
+pinned: false
+---

app.py

@@ -0,0 +1,591 @@
+import os
+import re
+import json
+import asyncio
+import itertools
+from typing import Any, Dict, List, Tuple, Optional
+
+import httpx
+from cryptography.fernet import Fernet
+from fastapi import FastAPI, Request, Depends, HTTPException
+from fastapi.responses import StreamingResponse, Response
+from fastapi.middleware.cors import CORSMiddleware
+
+# ----------------------------------------------------------------------------
+# Config from secrets
+# ----------------------------------------------------------------------------
+KEY = os.environ.get("KEY", "").strip()
+AES_KEY = os.environ.get("AES_KEY", "").strip()
+
+if not KEY:
+    raise RuntimeError("KEY secret is not set")
+if not AES_KEY:
+    raise RuntimeError("AES_KEY secret is not set")
+
+# ----------------------------------------------------------------------------
+# Decrypt models.json at runtime. The plaintext is never on disk.
+# ----------------------------------------------------------------------------
+_fernet = Fernet(AES_KEY.encode() if isinstance(AES_KEY, str) else AES_KEY)
+with open("models.json.enc", "rb") as _f:
+    MODELS: Dict[str, Dict[str, Any]] = json.loads(_fernet.decrypt(_f.read()))
+
+# Per-model round-robin state
+_RR: Dict[str, itertools.cycle] = {
+    name: itertools.cycle(range(len(cfg["targets"])))
+    for name, cfg in MODELS.items()
+}
+_RR_LOCKS: Dict[str, asyncio.Lock] = {name: asyncio.Lock() for name in MODELS}
+
+# Unique (url, key) endpoints across all models, for /health probing
+def _unique_endpoints() -> List[Tuple[str, str]]:
+    seen = set(); out = []
+    for cfg in MODELS.values():
+        for t in cfg["targets"]:
+            k = (t["url"], t["key"])
+            if k not in seen:
+                seen.add(k); out.append(k)
+    return out
+
+ENDPOINTS = _unique_endpoints()
+
+# ----------------------------------------------------------------------------
+# Auth -- accept the key from any header a harness might send:
+# Authorization: Bearer KEY (OpenAI SDK, LiteLLM, ...)
+# x-api-key: KEY                  (Anthropic SDK, Claude Code)
+# api-key: KEY                    (Azure-style)
+# x-goog-api-key: KEY             (Google AI SDK)
+# ----------------------------------------------------------------------------
+_AUTH_HEADERS = ("authorization", "x-api-key", "api-key", "x-goog-api-key")
+
+
+def _provided_key(request: Request) -> Optional[str]:
+    for h in _AUTH_HEADERS:
+        v = request.headers.get(h)
+        if not v:
+            continue
+        v = v.strip()
+        if v.lower().startswith("bearer "):
+            v = v[7:].strip()
+        if v:
+            return v
+    # allow ?api_key= / ?key= query too
+    return request.query_params.get("api_key") or request.query_params.get("key")
+
+
+async def require_auth(request: Request):
+    if _provided_key(request) == KEY:
+        return True
+    raise HTTPException(status_code=401, detail="invalid api key")
+
+# ----------------------------------------------------------------------------
+# App -- docs/redoc/openapi fully disabled and explicitly blocked
+# ----------------------------------------------------------------------------
+app = FastAPI(
+    title="router",
+    docs_url=None,
+    redoc_url=None,
+    openapi_url=None,
+)
+
+# CORS -- browser harnesses (Continue, web UIs, etc.) send preflight OPTIONS.
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=False,
+    allow_methods=["*"],
+    allow_headers=["*"],
+    expose_headers=["*"],
+)
+
+@app.get("/docs", include_in_schema=False)
+@app.get("/redoc", include_in_schema=False)
+@app.get("/openapi.json", include_in_schema=False)
+@app.get("/docs.json", include_in_schema=False)
+async def _block_docs():
+    raise HTTPException(status_code=404)
+
+# ----------------------------------------------------------------------------
+# Helpers
+# ----------------------------------------------------------------------------
+# ----------------------------------------------------------------------------
+# Cooldown / circuit-breaker: when a target fails with a retryable error,
+# it is banned for BAN_TTL seconds. _pick_target skips banned targets and
+# only falls back to them if every target is currently banned (so a cold
+# start or all-banned state still serves the request).
+# ----------------------------------------------------------------------------
+import time as _time
+_BAN: Dict[Tuple[str, str], float] = {}
+# Tiered bans: 1113 = dead account (rarely recovers); 429 = transient;
+# connect/5xx = probably transient. Long bans stop the prober from
+# re-testing confirmed-dead keys every cycle.
+_BAN_TTL_LONG = 3600.0    # 1113 insufficient balance / 401 / 403
+_BAN_TTL_MED = 120.0      # connect error / 5xx
+_BAN_TTL_SHORT = 20.0     # 429 rate limit (recovers fast)
+_PROBE_INTERVAL = 600.0  # re-probe every 10 min (cheaper than 4)
+_PROBE_CONCURRENCY = 40   # fast first-cycle classification (~15s for 683)
+
+
+# Shared connection pool -- ONE persistent client reused by every request
+# and by the prober. This eliminates the TLS handshake cost (~200-400ms)
+# that a per-request httpx.AsyncClient pays on every retry. Keep-alive keeps
+# the TCP+TLS connection to api.z.ai / bigmodel.cn warm.
+_CLIENT: Optional[httpx.AsyncClient] = None
+
+
+async def _client() -> httpx.AsyncClient:
+    global _CLIENT
+    if _CLIENT is None or _CLIENT.is_closed:
+        _CLIENT = httpx.AsyncClient(
+            timeout=httpx.Timeout(600.0, connect=10.0, read=600.0),
+            limits=httpx.Limits(
+                max_connections=300,
+                max_keepalive_connections=80,
+                keepalive_expiry=300.0,
+            ),
+        )
+    return _CLIENT
+
+
+def _ban(target, ttl=_BAN_TTL_MED):
+    _BAN[(target["url"], target["key"])] = _time.monotonic() + ttl
+
+
+def _is_banned(target) -> bool:
+    exp = _BAN.get((target["url"], target["key"]))
+    return exp is not None and exp > _time.monotonic()
+
+
+# ----------------------------------------------------------------------------
+# Background balance prober. /models lies -- it returns 200 even for drained
+# keys. Only a real 1-token chat reveals the 1113 (insufficient balance)
+# error. This task continuously classifies every key, banning drained ones
+# and keeping the good pool hot so requests hit a good key on the first try.
+# Cost: ~1 token per good key per cycle (drained keys error out at 0 cost).
+# ----------------------------------------------------------------------------
+async def _probe_balance(client: httpx.AsyncClient, url: str, key: str) -> bool:
+    try:
+        r = await client.post(
+            url.rstrip("/") + "/chat/completions",
+            headers={"Authorization": "Bearer " + key, "Content-Type": "application/json"},
+            json={"model": "glm-4.6", "messages": [{"role": "user", "content": "."}], "max_tokens": 1},
+            timeout=15.0,
+        )
+    except Exception:
+        return False
+    if r.status_code != 200:
+        return False
+    return not _is_retryable_error(r.content)
+
+
+async def _classify_all():
+    """One-time bootstrap: probe EVERY key once to populate the ban table fast
+    (~15s at concurrency 40) so the first real request doesn't walk 640 dead
+    keys. Good keys may briefly 429 but recover within a minute."""
+    sem = asyncio.Semaphore(_PROBE_CONCURRENCY)
+    c = await _client()
+    async def one(u, k):
+        async with sem:
+            ok = await _probe_balance(c, u, k)
+        if ok:
+            _BAN.pop((u, k), None)
+        else:
+            _BAN[(u, k)] = _time.monotonic() + _BAN_TTL_LONG
+    await asyncio.gather(*[one(u, k) for u, k in ENDPOINTS])
+
+
+async def _probe_all():
+    """Re-probe BANNED keys whose ban is about to expire, to catch recoveries.
+    Working/un-banned keys are never probed -- live traffic validates them and
+    the prober would just 429 them. This is a recovery sweep, not a full scan."""
+    sem = asyncio.Semaphore(_PROBE_CONCURRENCY)
+    c = await _client()
+    now = _time.monotonic()
+    # Keys banned within the next PROBE_INTERVAL + buffer are candidates.
+    horizon = now + _PROBE_INTERVAL + 60
+    todo = [(u, k) for (u, k) in ENDPOINTS
+            if _BAN.get((u, k), 0) <= horizon and _BAN.get((u, k), 0) > now]
+
+    async def one(u, k):
+        async with sem:
+            ok = await _probe_balance(c, u, k)
+        if ok:
+            _BAN.pop((u, k), None)   # recovered -> available again
+        else:
+            _BAN[(u, k)] = _time.monotonic() + _BAN_TTL_LONG
+    if todo:
+        await asyncio.gather(*[one(u, k) for u, k in todo])
+
+
+async def _prober_loop():
+    # Bootstrap: classify all keys once so the ban table is populated.
+    try:
+        await _classify_all()
+    except Exception:
+        pass
+    # Steady state: only sweep recovering banned keys, never touch working
+    # ones (live traffic validates those).
+    while True:
+        await asyncio.sleep(_PROBE_INTERVAL)
+        try:
+            await _probe_all()
+        except Exception:
+            pass
+
+
+@app.on_event("startup")
+async def _start_prober():
+    await _client()   # warm the connection pool
+    asyncio.create_task(_prober_loop())
+
+
+@app.on_event("shutdown")
+async def _close_client():
+    global _CLIENT
+    if _CLIENT is not None:
+        await _CLIENT.aclose()
+        _CLIENT = None
+
+
+async def _pick_target(model_name: str):
+    """Round-robin next target, skipping banned ones unless all are banned."""
+    cfg = MODELS[model_name]
+    targets = cfg["targets"]
+    n = len(targets)
+    async with _RR_LOCKS[model_name]:
+        # First pass: find the next non-banned target.
+        for _ in range(n):
+            idx = next(_RR[model_name])
+            t = targets[idx]
+            if not _is_banned(t):
+                return t
+        # Everything is banned -- return the round-robin pick anyway so the
+        # request still goes out. Ban timestamps will have started expiring.
+        idx = next(_RR[model_name])
+        return targets[idx]
+
+# hop-by-hop / router-only headers we never copy upstream or downstream
+_HOP = {
+    "host", "content-length", "transfer-encoding", "connection",
+    "keep-alive", "proxy-authenticate", "proxy-authorization",
+    "te", "trailers", "upgrade",
+}
+
+
+def _resolve_model(payload: Optional[dict], request: Request) -> Optional[str]:
+    """Find the requested model from JSON body, query, or header."""
+    if isinstance(payload, dict):
+        m = payload.get("model")
+        if isinstance(m, str) and m:
+            return m
+    m = request.query_params.get("model")
+    if m:
+        return m
+    m = request.headers.get("x-router-model")
+    if m:
+        return m
+    return None
+
+# GLM/Zhipu per-key error codes that mean "this key is unusable, try another":
+# 1111/1112 invalid or expired key, 1113 insufficient balance,
+# 1114/1115/1116/1117 quota / resource-pack exhausted.
+_RETRYABLE_CODES = {"1111", "1112", "1113", "1114", "1115", "1116", "1117", "1261"}
+
+
+# ----------------------------------------------------------------------------
+# Protocol / harness detection
+# ----------------------------------------------------------------------------
+# GLM exposes two native protocol surfaces:
+#   OpenAI-compatible:   <host>/api/paas/v4/<path>   (and /api/coding/paas/v4)
+#   Anthropic-compatible: <host>/api/anthropic/<path>
+# We detect which harness is calling us from the request path and route to the
+# matching upstream base, so OpenAI SDK, Anthropic SDK / Claude Code, LiteLLM,
+# LangChain, Cline, Continue, aider, etc. all work with zero translation.
+
+def _detect_format(path: str) -> str:
+    p = path.lower().lstrip("/")
+    if (
+        p.startswith("v1/messages")
+        or p == "messages"
+        or p.startswith("messages/")
+        or "count_tokens" in p
+    ):
+        return "anthropic"
+    if p.startswith("v1beta/") or ":generatecontent" in p or ":streamgeneratecontent" in p:
+        return "gemini"
+    return "openai"
+
+
+_ANTR_RE = re.compile(r"^(https?://[^/]+/api)(/.*)?$")
+
+
+def _target_base(target_url: str, fmt: str) -> str:
+    """Map an OpenAI-format target base to the right base for the protocol."""
+    base = target_url.rstrip("/")
+    if fmt == "anthropic":
+        m = _ANTR_RE.match(base)
+        if m:
+            return m.group(1) + "/anthropic"
+        return base
+    # openai + gemini (gemini falls back to openai base, best-effort)
+    return base
+
+
+def _forward_path(path: str, fmt: str) -> str:
+    """Compute the path segment to append to the upstream base."""
+    p = path.lstrip("/")
+    if fmt == "openai":
+        # base already ends in /v4; drop a leading v1/ from the harness path
+        if p.startswith("v1/"):
+            p = p[3:]
+        return p
+    if fmt == "anthropic":
+        # upstream anthropic base has no version; the harness's /v1/ stays
+        if not p.startswith("v1/"):
+            p = "v1/" + p
+        return p
+    return p  # gemini passthrough
+
+
+_GEM_MODEL_RE = re.compile(r"/models/([^/:]+)", re.I)
+
+
+def _extract_model_gemini(path: str) -> Optional[str]:
+    m = _GEM_MODEL_RE.search("/" + path)
+    return m.group(1) if m else None
+
+
+
+def _is_retryable_error(buf: bytes) -> bool:
+    """Return True if buf is a GLM error JSON we should retry on another key.
+
+    Handles both protocol shapes:
+      OpenAI:    {"error":{"code":"1113","message":"..."}}
+      Anthropic: {"type":"error","error":{"type":"...","code":"1113","message":"..."}}
+    """
+    if not buf or buf[:1] != b"{":
+        return False
+    try:
+        j = json.loads(buf)
+    except Exception:
+        return False
+    if not isinstance(j, dict):
+        return False
+    # Both shapes carry the GLM code under error.code or top-level code.
+    err = j.get("error") if isinstance(j.get("error"), dict) else {}
+    code = str(err.get("code") or j.get("code") or "")
+    if code in _RETRYABLE_CODES:
+        return True
+    msg = str(err.get("message") or j.get("message") or "").lower()
+    if any(s in msg for s in ("balance", "quota", "余额", "配额", "insufficient")):
+        return True
+    # Anthropic rate_limit_error / overloaded -> try another key.
+    etype = str(err.get("type") or "").lower()
+    if etype in ("rate_limit_error", "overloaded_error"):
+        return True
+    return False
+
+# ----------------------------------------------------------------------------
+# Health -- instant. Reads the ban table maintained by the background
+# prober. No outbound calls (the old version fired 683 probes per request
+# and starved the event loop).
+# ----------------------------------------------------------------------------
+@app.get("/health")
+@app.get("/healthz")
+async def health():
+    now_mono = _time.monotonic()
+    banned = sum(1 for exp in _BAN.values() if exp > now_mono)
+    return {
+        "total": len(ENDPOINTS),
+        "available": len(ENDPOINTS) - banned,
+        "banned": banned,
+        "models": len(MODELS),
+    }
+
+# ----------------------------------------------------------------------------
+# Model list
+# ----------------------------------------------------------------------------
+@app.get("/v1/models")
+async def list_models():
+    return {
+        "object": "list",
+        "data": [
+            {"id": name, "object": "model", "owned_by": "router"}
+            for name in MODELS
+        ],
+    }
+
+@app.get("/")
+async def root():
+    now_mono = _time.monotonic()
+    banned = sum(1 for exp in _BAN.values() if exp > now_mono)
+    return {
+        "ok": True,
+        "models": len(MODELS),
+        "endpoints": len(ENDPOINTS),
+        "available": len(ENDPOINTS) - banned,
+        "banned": banned,
+    }
+
+# ----------------------------------------------------------------------------
+# Transparent catch-all proxy. Mounted under /{path:path} so harnesses that
+# include /v1/ (OpenAI/Anthropic) AND those that drop it both work. Public
+# routes above (/health, /v1/models, /) are matched before this catch-all.
+# ----------------------------------------------------------------------------
+@app.api_route(
+    "/{path:path}",
+    methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"],
+    dependencies=[Depends(require_auth)],
+)
+async def proxy(path: str, request: Request):
+    fmt = _detect_format(path)
+
+    raw = await request.body()
+    ctype = request.headers.get("content-type", "")
+
+    # Parse body only if JSON -- otherwise forward bytes verbatim (multipart,
+    # binary, etc.). Tools / messages / stream / thinking all live in JSON
+    # and are passed through untouched.
+    payload: Optional[dict] = None
+    body_to_send: bytes = raw
+    is_json = "application/json" in ctype.lower() or ctype == ""
+    if is_json and raw:
+        try:
+            payload = json.loads(raw)
+            if not isinstance(payload, dict):
+                payload = None
+        except Exception:
+            payload = None
+
+    # Model resolution works across all harnesses:
+    #   OpenAI/Anthropic: "model" in JSON body
+    #   Gemini: model in URL path (v1beta/models/<model>:generateContent)
+    model_name = _resolve_model(payload, request)
+    if model_name is None and fmt == "gemini":
+        model_name = _extract_model_gemini(path)
+    if not model_name or model_name not in MODELS:
+        raise HTTPException(
+            status_code=400,
+            detail=f"unknown model: {model_name!r}; available: {list(MODELS)}",
+        )
+
+    cfg = MODELS[model_name]
+    n_targets = len(cfg["targets"])
+    last_err = "no targets"
+    fwd_rel = _forward_path(path, fmt)
+
+    for _ in range(n_targets):
+        target = await _pick_target(model_name)
+        base = _target_base(target["url"], fmt)
+
+        # Build outbound body: swap ONLY the model field when JSON. For Gemini
+        # the model lives in the URL path; since upstream_model == model here
+        # we forward the path verbatim.
+        if isinstance(payload, dict):
+            out = dict(payload)
+            out["model"] = target["upstream_model"]
+            send_bytes = json.dumps(out, ensure_ascii=False).encode()
+        else:
+            send_bytes = body_to_send
+
+        url = base.rstrip("/") + "/" + fwd_rel.lstrip("/")
+
+        # Forward every client header transparently except hop-by-hop + the
+        # auth headers we must override for the upstream.
+        fwd = {
+            k: v for k, v in request.headers.items()
+            if k.lower() not in _HOP
+            and k.lower() not in _AUTH_HEADERS
+        }
+        # Anthropic upstream wants x-api-key, OpenAI/Gemini want Bearer.
+        if fmt == "anthropic":
+            fwd["x-api-key"] = target["key"]
+        else:
+            fwd["Authorization"] = "Bearer " + target["key"]
+        if send_bytes:
+            fwd["content-length"] = str(len(send_bytes))
+        if "content-type" not in {k.lower() for k in fwd} and ctype:
+            fwd["Content-Type"] = ctype
+
+        query = str(request.url.query)
+
+        client = await _client()
+        try:
+            req = client.build_request(
+                request.method, url,
+                headers=fwd,
+                content=send_bytes if send_bytes else None,
+                params=query or None,
+            )
+            resp = await client.send(req, stream=True)
+        except Exception as e:
+            _ban(target, _BAN_TTL_MED)
+            last_err = f"connect: {e}"
+            continue
+
+        # Classify the upstream response. The key insight: GLM serves
+        # balance-drained errors (code 1113) with HTTP 429 status, NOT a real
+        # rate-limit. So we must inspect the body before choosing a ban TTL:
+        #   1113/quota in body -> dead account, LONG ban (don't re-walk it)
+        #   genuine 429         -> transient, SHORT ban
+        #   5xx / connect err   -> transient, MED ban
+        #   401/403             -> dead auth, LONG ban
+        ctype_resp = resp.headers.get("content-type", "")
+        is_sse = ctype_resp.lower().startswith("text/event-stream")
+
+        if resp.status_code >= 400 or ("application/json" in ctype_resp.lower() and not is_sse):
+            body_buf = await resp.aread()
+            await resp.aclose()
+            retryable = _is_retryable_error(body_buf)
+            if retryable:
+                # 1113 balance / quota / drained -> dead key, don't revisit
+                _ban(target, _BAN_TTL_LONG)
+                last_err = f"drained: {body_buf[:120]!r}"
+                continue
+            if resp.status_code == 429:
+                # genuine rate limit (no 1113) -> recovers fast
+                _ban(target, _BAN_TTL_SHORT)
+                last_err = f"rate-limited: {body_buf[:120]!r}"
+                continue
+            if resp.status_code in (401, 403):
+                _ban(target, _BAN_TTL_LONG)
+                last_err = f"auth {resp.status_code}: {body_buf[:120]!r}"
+                continue
+            if resp.status_code >= 500:
+                _ban(target, _BAN_TTL_MED)
+                last_err = f"upstream {resp.status_code}: {body_buf[:120]!r}"
+                continue
+            # Non-retryable 4xx (bad request, unknown model, ...) -> return
+            resp_headers = {
+                k: v for k, v in resp.headers.items()
+                if k.lower() not in _HOP and k.lower() != "content-encoding"
+            }
+            return Response(
+                content=body_buf,
+                status_code=resp.status_code,
+                headers=resp_headers,
+                media_type=resp.headers.get("content-type"),
+            )
+
+        # Pass through response headers (content-type, x-request-id,
+        # anthropic-*, caching headers, etc.) untouched.
+        resp_headers = {
+            k: v for k, v in resp.headers.items()
+            if k.lower() not in _HOP
+        }
+
+        async def gen():
+            try:
+                async for chunk in resp.aiter_raw():
+                    if chunk:
+                        yield chunk
+            finally:
+                await resp.aclose()
+
+        return StreamingResponse(
+            gen(),
+            status_code=resp.status_code,
+            headers=resp_headers,
+            media_type=resp.headers.get("content-type"),
+        )
+
+    raise HTTPException(status_code=502, detail=f"all targets failed: {last_err}")

requirements.txt

@@ -0,0 +1,4 @@
+fastapi>=0.110
+uvicorn[standard]>=0.27
+httpx>=0.27
+cryptography>=42