Keep Gemini failures model scoped

.env.example

@@ -173,9 +173,10 @@ LLM_API_KEY_FALLBACK_ENABLED=true
 # overloading one key when many requests arrive in parallel.
 # KEY_MAX_INFLIGHT_PER_KEY=3
 #
-# Safety lease for in-flight counters (default: 5 min). Prevents abandoned
-# streaming/undici requests from making keys look permanently saturated.
-# KEY_INFLIGHT_TTL_MS=300000
+# Safety lease for in-flight counters (default: 30s). If a picked key gets no
+# provider headers/completion/error before this, it is marked transient and
+# sticky mode rotates away instead of leaving permanent pending counts.
+# KEY_INFLIGHT_TTL_MS=30000
 #
 # Max request-body bytes to inspect for model names on streaming
 # OpenAI-compatible Gemini calls (default: 256 KiB).
@@ -191,6 +192,7 @@ LLM_API_KEY_FALLBACK_ENABLED=true
 # model-scoped and per-request round-robin can burn 2-3 keys for one chat turn.
 # KEY_STICKY_UNTIL_FAILURE=true
 # KEY_STICKY_PROVIDERS=gemini
+# KEY_STICKY_SCOPE=auto   # auto = per-model for Gemini, provider-level for others
 #
 # Optional auto-retry count for fetch requests on retryable errors/statuses.
 # Default 0 means one upstream attempt per caller request; set 1-2 to opt in.

README.md

@@ -296,11 +296,12 @@ Optional tuning:
 - `KEY_PERM_SUSPEND_MS` (default `57600000`) — long suspend duration for exhausted/auth-invalid keys (**capped at 16h max**).
 - `KEY_FAILURE_DECAY_MS` (default `900000`) — recent-failure decay window used to deprioritize keys.
 - `KEY_MAX_INFLIGHT_PER_KEY` (default `3`) — soft concurrent request cap per key.
-- `KEY_INFLIGHT_TTL_MS` (default `300000`) — safety lease for in-flight counters, preventing stale undici streams from making keys look permanently saturated.
+- `KEY_INFLIGHT_TTL_MS` (default `30000`) — safety lease for picked keys with no provider headers/completion/error; stale picks are marked transient so sticky keys can rotate.
 - `KEY_MODEL_SNIFF_MAX_BYTES` (default `262144`) — max request-body bytes to inspect for model names on streaming OpenAI-compatible Gemini calls.
 - `KEY_ERROR_BODY_SNIFF_MAX_BYTES` (default `65536`) — max error-response bytes to inspect so provider quota/rate bodies such as 403 quota errors are scoped correctly instead of being treated as permanent auth failures.
 - `KEY_STICKY_UNTIL_FAILURE` (default `true`) — keep sticky providers on one key until that key fails/exhausts.
 - `KEY_STICKY_PROVIDERS` (default `gemini`) — comma-separated provider names that should use sticky key selection instead of per-request round-robin.
+- `KEY_STICKY_SCOPE` (default `auto`) — `auto` uses per-model sticky buckets for Gemini/per-model providers and provider-level buckets for others; set `provider` or `model` to override.
 - `KEY_FETCH_MAX_RETRIES` (default `0`) — optional auto-retry count for retryable failures on **GET/HEAD/OPTIONS/POST** with a different key. Default `0` means the rotator does **not** spend extra upstream attempts for a single caller request.
 - `KEY_FETCH_RETRY_BASE_DELAY_MS` (default `250`) — base delay for retry backoff (respects `Retry-After`, capped to 10s).
 - `KEY_ROTATOR_ASSERT_NO_EXTRA_CALLS=true` — optional diagnostic warning if a single caller fetch creates more than one upstream provider attempt.

env-builder.js

@@ -612,9 +612,9 @@ const FIELDS = [
     "k": "KEY_INFLIGHT_TTL_MS",
     "lbl": "Key rotation in-flight safety lease (ms)",
     "type": "text",
-    "ph": "300000",
+    "ph": "30000",
     "tag": "advanced",
-    "help": "Clears stale in-flight counters if a streaming/undici request never completes cleanly. Default: 5 minutes."
+    "help": "Marks picked keys stale if no provider headers/completion/error are observed. Default: 30 seconds."
   },
 {
     "g": "Plugins",
@@ -656,6 +656,16 @@ const FIELDS = [
     "tag": "advanced",
     "help": "Provider names that should reuse one key until failure/quota exhaustion. Default: gemini."
   },
+  {
+    "g": "Plugins",
+    "icon": "🔄",
+    "k": "KEY_STICKY_SCOPE",
+    "lbl": "Sticky key scope (auto/provider/model)",
+    "type": "text",
+    "ph": "auto",
+    "tag": "advanced",
+    "help": "Default auto uses per-model sticky buckets for Gemini/per-model providers and provider-level buckets for others."
+  },
 {
     "g": "Plugins",
     "icon": "🔄",

key-rotator-manager.html

@@ -5,7 +5,7 @@
   <meta name="viewport" content="width=device-width,initial-scale=1" />
   <title>HuggingClaw · API Key Rotator</title>
   <style>
-    :root{color-scheme:dark;--bg:#070711;--panel:#111120;--panel2:#17172a;--line:#292945;--text:#f8f7ff;--muted:#9892b8;--soft:#c7c2e6;--good:#22c55e;--warn:#f5c542;--bad:#fb7185;--blue:#60a5fa;--violet:#a78bfa}*{box-sizing:border-box}body{margin:0;min-height:100vh;background:radial-gradient(circle at top left,#25145a 0,#070711 34%,#070711 100%);font-family:Inter,ui-sans-serif,system-ui,-apple-system,sans-serif;color:var(--text);font-size:13px}main{width:min(1280px,calc(100% - 28px));margin:0 auto;padding:28px 0 44px}.top{display:flex;align-items:flex-start;justify-content:space-between;gap:18px;margin-bottom:18px}.eyebrow{font-size:.7rem;letter-spacing:.18em;text-transform:uppercase;color:var(--muted);font-weight:900}h1{margin:6px 0 8px;font-size:clamp(1.6rem,4vw,2.7rem);line-height:1}.sub{color:var(--soft);max-width:860px;line-height:1.55}.actions{display:flex;gap:10px;flex-wrap:wrap}.btn{border:1px solid var(--line);background:rgba(255,255,255,.06);color:var(--text);border-radius:11px;padding:10px 14px;text-decoration:none;font-weight:850;cursor:pointer}.btn.primary{background:#fff;color:#050510}.btn:hover{filter:brightness(1.08)}.grid{display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:12px;margin:18px 0}.card{background:linear-gradient(180deg,rgba(255,255,255,.055),rgba(255,255,255,.025));border:1px solid var(--line);border-radius:18px;padding:16px;box-shadow:0 18px 45px rgba(0,0,0,.22)}.metric-title{color:var(--muted);text-transform:uppercase;letter-spacing:.16em;font-size:.66rem;font-weight:900}.metric-value{font-size:1.65rem;font-weight:950;margin-top:8px}.metric-detail{color:var(--muted);margin-top:6px;line-height:1.45}.ok{color:var(--good)}.warn{color:var(--warn)}.bad{color:var(--bad)}.blue{color:var(--blue)}.layout{display:grid;grid-template-columns:340px minmax(0,1fr);gap:14px}.panel-title{display:flex;justify-content:space-between;align-items:center;gap:10px;margin-bottom:12px}.panel-title h2{font-size:1rem;margin:0}.pill{display:inline-flex;align-items:center;gap:6px;border:1px solid var(--line);border-radius:999px;padding:5px 9px;color:var(--soft);background:rgba(255,255,255,.035);font-size:.72rem;font-weight:850}.dot{width:7px;height:7px;border-radius:50%;background:var(--muted)}.dot.live{background:var(--good);box-shadow:0 0 15px var(--good)}.providers{display:flex;flex-direction:column;gap:9px}.provider{border:1px solid var(--line);border-radius:14px;background:rgba(0,0,0,.12);padding:12px;cursor:pointer}.provider.active{border-color:var(--blue);box-shadow:0 0 0 1px rgba(96,165,250,.25)}.provider-top{display:flex;justify-content:space-between;align-items:center;gap:12px}.provider-name{font-weight:950}.provider-meta{color:var(--muted);font-size:.78rem;margin-top:5px}.bar{height:8px;background:#22223a;border-radius:999px;overflow:hidden;margin-top:10px}.bar>span{display:block;height:100%;background:linear-gradient(90deg,var(--violet),var(--blue));border-radius:999px}.toolbar{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px}.input{flex:1;min-width:180px;border:1px solid var(--line);background:#0c0c18;color:var(--text);border-radius:11px;padding:10px 12px;outline:none}.select{border:1px solid var(--line);background:#0c0c18;color:var(--text);border-radius:11px;padding:10px 12px}.toggle{display:flex;align-items:center;gap:7px;color:var(--soft);font-weight:800}.events{display:flex;flex-direction:column;gap:9px;max-height:620px;overflow:auto;padding-right:4px}.event{border:1px solid var(--line);border-left-width:4px;border-radius:14px;background:rgba(0,0,0,.16);padding:12px;display:grid;grid-template-columns:155px minmax(0,1fr) auto;gap:12px}.event.pick,.event.sticky_pick,.event.pick_retry_fresh,.event.model_detected{border-left-color:var(--blue)}.event.success{border-left-color:var(--good)}.event.rate_limited,.event.auth_failed,.event.all_suspended_pick,.event.all_suspended_withheld{border-left-color:var(--warn)}.event.network_retryable,.event.transient_status,.event.saturated_reuse,.event.sticky_saturated_reuse,.event.inflight_timeout{border-left-color:var(--bad)}.time{color:var(--muted);font-variant-numeric:tabular-nums}.etype{font-weight:950}.msg{color:var(--soft);line-height:1.45;word-break:break-word}.key,.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:#e9d5ff}.empty{padding:34px;text-align:center;color:var(--muted);border:1px dashed var(--line);border-radius:16px}.foot{color:var(--muted);margin-top:16px;line-height:1.55}.kbd{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;background:#23233a;border:1px solid #363653;border-radius:7px;padding:2px 6px;color:var(--text)}.detail-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:10px;margin-bottom:12px}.mini{border:1px solid var(--line);border-radius:13px;padding:11px;background:rgba(0,0,0,.12)}.mini b{display:block;font-size:1.15rem;margin-top:4px}.key-table-wrap{overflow:auto}.key-table{width:100%;border-collapse:separate;border-spacing:0 8px;min-width:780px}.key-table th{color:var(--muted);font-size:.68rem;text-align:left;text-transform:uppercase;letter-spacing:.12em}.key-table td{background:rgba(0,0,0,.15);border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:10px;vertical-align:top}.key-table td:first-child{border-left:1px solid var(--line);border-radius:12px 0 0 12px}.key-table td:last-child{border-right:1px solid var(--line);border-radius:0 12px 12px 0}.status{font-weight:900}.status.used{color:var(--good)}.status.unused{color:var(--muted)}.model-chip{display:inline-flex;margin:2px 4px 2px 0;padding:4px 7px;border:1px solid var(--line);border-radius:999px;color:var(--soft);background:rgba(255,255,255,.035);font-size:.72rem}.section{margin-top:14px}.session-note{border:1px solid rgba(96,165,250,.28);background:rgba(96,165,250,.08);border-radius:14px;padding:12px;color:var(--soft);margin-bottom:12px}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}@media(max-width:1050px){.grid{grid-template-columns:repeat(2,minmax(0,1fr))}.detail-grid{grid-template-columns:repeat(2,minmax(0,1fr))}.layout{grid-template-columns:1fr}.top{flex-direction:column}.event{grid-template-columns:1fr}.events{max-height:none}}@media(max-width:620px){.grid,.detail-grid{grid-template-columns:1fr}}
+    :root{color-scheme:dark;--bg:#070711;--panel:#111120;--panel2:#17172a;--line:#292945;--text:#f8f7ff;--muted:#9892b8;--soft:#c7c2e6;--good:#22c55e;--warn:#f5c542;--bad:#fb7185;--blue:#60a5fa;--violet:#a78bfa}*{box-sizing:border-box}body{margin:0;min-height:100vh;background:radial-gradient(circle at top left,#25145a 0,#070711 34%,#070711 100%);font-family:Inter,ui-sans-serif,system-ui,-apple-system,sans-serif;color:var(--text);font-size:13px}main{width:min(1280px,calc(100% - 28px));margin:0 auto;padding:28px 0 44px}.top{display:flex;align-items:flex-start;justify-content:space-between;gap:18px;margin-bottom:18px}.eyebrow{font-size:.7rem;letter-spacing:.18em;text-transform:uppercase;color:var(--muted);font-weight:900}h1{margin:6px 0 8px;font-size:clamp(1.6rem,4vw,2.7rem);line-height:1}.sub{color:var(--soft);max-width:860px;line-height:1.55}.actions{display:flex;gap:10px;flex-wrap:wrap}.btn{border:1px solid var(--line);background:rgba(255,255,255,.06);color:var(--text);border-radius:11px;padding:10px 14px;text-decoration:none;font-weight:850;cursor:pointer}.btn.primary{background:#fff;color:#050510}.btn:hover{filter:brightness(1.08)}.grid{display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:12px;margin:18px 0}.card{background:linear-gradient(180deg,rgba(255,255,255,.055),rgba(255,255,255,.025));border:1px solid var(--line);border-radius:18px;padding:16px;box-shadow:0 18px 45px rgba(0,0,0,.22)}.metric-title{color:var(--muted);text-transform:uppercase;letter-spacing:.16em;font-size:.66rem;font-weight:900}.metric-value{font-size:1.65rem;font-weight:950;margin-top:8px}.metric-detail{color:var(--muted);margin-top:6px;line-height:1.45}.ok{color:var(--good)}.warn{color:var(--warn)}.bad{color:var(--bad)}.blue{color:var(--blue)}.layout{display:grid;grid-template-columns:340px minmax(0,1fr);gap:14px}.panel-title{display:flex;justify-content:space-between;align-items:center;gap:10px;margin-bottom:12px}.panel-title h2{font-size:1rem;margin:0}.pill{display:inline-flex;align-items:center;gap:6px;border:1px solid var(--line);border-radius:999px;padding:5px 9px;color:var(--soft);background:rgba(255,255,255,.035);font-size:.72rem;font-weight:850}.dot{width:7px;height:7px;border-radius:50%;background:var(--muted)}.dot.live{background:var(--good);box-shadow:0 0 15px var(--good)}.providers{display:flex;flex-direction:column;gap:9px}.provider{border:1px solid var(--line);border-radius:14px;background:rgba(0,0,0,.12);padding:12px;cursor:pointer}.provider.active{border-color:var(--blue);box-shadow:0 0 0 1px rgba(96,165,250,.25)}.provider-top{display:flex;justify-content:space-between;align-items:center;gap:12px}.provider-name{font-weight:950}.provider-meta{color:var(--muted);font-size:.78rem;margin-top:5px}.bar{height:8px;background:#22223a;border-radius:999px;overflow:hidden;margin-top:10px}.bar>span{display:block;height:100%;background:linear-gradient(90deg,var(--violet),var(--blue));border-radius:999px}.toolbar{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px}.input{flex:1;min-width:180px;border:1px solid var(--line);background:#0c0c18;color:var(--text);border-radius:11px;padding:10px 12px;outline:none}.select{border:1px solid var(--line);background:#0c0c18;color:var(--text);border-radius:11px;padding:10px 12px}.toggle{display:flex;align-items:center;gap:7px;color:var(--soft);font-weight:800}.events{display:flex;flex-direction:column;gap:9px;max-height:620px;overflow:auto;padding-right:4px}.event{border:1px solid var(--line);border-left-width:4px;border-radius:14px;background:rgba(0,0,0,.16);padding:12px;display:grid;grid-template-columns:155px minmax(0,1fr) auto;gap:12px}.event.pick,.event.sticky_pick,.event.pick_retry_fresh,.event.model_detected{border-left-color:var(--blue)}.event.success{border-left-color:var(--good)}.event.rate_limited,.event.auth_failed,.event.all_suspended_pick,.event.all_suspended_withheld{border-left-color:var(--warn)}.event.network_retryable,.event.transient_status,.event.saturated_reuse,.event.sticky_saturated_reuse,.event.sticky_saturated_rotate,.event.inflight_timeout{border-left-color:var(--bad)}.time{color:var(--muted);font-variant-numeric:tabular-nums}.etype{font-weight:950}.msg{color:var(--soft);line-height:1.45;word-break:break-word}.key,.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:#e9d5ff}.empty{padding:34px;text-align:center;color:var(--muted);border:1px dashed var(--line);border-radius:16px}.foot{color:var(--muted);margin-top:16px;line-height:1.55}.kbd{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;background:#23233a;border:1px solid #363653;border-radius:7px;padding:2px 6px;color:var(--text)}.detail-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:10px;margin-bottom:12px}.mini{border:1px solid var(--line);border-radius:13px;padding:11px;background:rgba(0,0,0,.12)}.mini b{display:block;font-size:1.15rem;margin-top:4px}.key-table-wrap{overflow:auto}.key-table{width:100%;border-collapse:separate;border-spacing:0 8px;min-width:780px}.key-table th{color:var(--muted);font-size:.68rem;text-align:left;text-transform:uppercase;letter-spacing:.12em}.key-table td{background:rgba(0,0,0,.15);border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:10px;vertical-align:top}.key-table td:first-child{border-left:1px solid var(--line);border-radius:12px 0 0 12px}.key-table td:last-child{border-right:1px solid var(--line);border-radius:0 12px 12px 0}.status{font-weight:900}.status.used{color:var(--good)}.status.unused{color:var(--muted)}.model-chip{display:inline-flex;margin:2px 4px 2px 0;padding:4px 7px;border:1px solid var(--line);border-radius:999px;color:var(--soft);background:rgba(255,255,255,.035);font-size:.72rem}.section{margin-top:14px}.session-note{border:1px solid rgba(96,165,250,.28);background:rgba(96,165,250,.08);border-radius:14px;padding:12px;color:var(--soft);margin-bottom:12px}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}@media(max-width:1050px){.grid{grid-template-columns:repeat(2,minmax(0,1fr))}.detail-grid{grid-template-columns:repeat(2,minmax(0,1fr))}.layout{grid-template-columns:1fr}.top{flex-direction:column}.event{grid-template-columns:1fr}.events{max-height:none}}@media(max-width:620px){.grid,.detail-grid{grid-template-columns:1fr}}
   </style>
 </head>
 <body>
@@ -56,14 +56,16 @@
 let state={events:[],providers:[],runtime:{routes:[]},paused:false,selected:localStorage.getItem('hc.keyRotator.provider')||''};
 const $=id=>document.getElementById(id), HTML_ESCAPE={'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'};
 const PICK_TYPES=['pick','sticky_pick','pick_retry_fresh','all_suspended_pick','saturated_reuse','sticky_saturated_reuse'];
+const STALE_PENDING_MS=35000;
 function esc(v){return String(v??'').replace(/[&<>"']/g,ch=>HTML_ESCAPE[ch]);}
 function eventClass(v){return String(v||'event').replace(/[^a-z0-9_-]/gi,'_');}
 function fmtTime(ts){const d=new Date(ts);return isNaN(d)?'—':d.toLocaleString();}
 function isKeyEvent(e){return e.provider&&e.slot&&(e.key||e.type==='all_suspended_withheld');}
 function keyId(provider,slot,key){return `${provider}::${slot||''}::${key||''}`;}
 function eventText(e){const bits=[];if(e.provider)bits.push(e.provider);if(e.slot)bits.push(`#${e.slot}/${e.total}`);if(e.key)bits.push(e.key);if(e.model)bits.push(`model=${e.model}`);if(e.status)bits.push(`status=${e.status}`);if(e.errorStatus)bits.push(`error=${e.errorStatus}`);if(e.errorReason)bits.push(`reason=${e.errorReason}`);if(e.errorType)bits.push(`type=${e.errorType}`);if(e.waitMs)bits.push(`wait=${Math.round(e.waitMs/1000)}s`);if(e.code)bits.push(`code=${e.code}`);if(e.errorCode)bits.push(`errorCode=${e.errorCode}`);if(e.inflight)bits.push(`inflight=${e.inflight}/${e.maxInflight||'?'}`);return bits.join(' · ');}
-function emptyStat(provider,slot,total,key){return{provider,slot,total,key,picks:0,success:0,rate:0,retry:0,auth:0,transient:0,last:'',models:new Map()};}
-function buildStats(){const stats=new Map();for(const p of state.providers){for(const k of (p.keys||[]))stats.set(keyId(p.name,k.slot,k.key),emptyStat(p.name,k.slot,k.total,k.key));}for(const e of state.events){if(!isKeyEvent(e))continue;const id=keyId(e.provider,e.slot,e.key);if(!stats.has(id))stats.set(id,emptyStat(e.provider,e.slot,e.total,e.key||'***'));const s=stats.get(id);const isPick=PICK_TYPES.includes(e.type);const isOutcome=['success','rate_limited','network_retryable','auth_failed','transient_status'].includes(e.type);if(isPick)s.picks++;if(e.type==='success')s.success++;if(e.type==='rate_limited')s.rate++;if(e.type==='network_retryable')s.retry++;if(e.type==='auth_failed')s.auth++;if(e.type==='transient_status')s.transient++;s.last=e.ts||s.last;if(e.model){let m=s.models.get(e.model);if(!m)m={picks:0,success:0,rate:0,retry:0,auth:0,transient:0,observed:false};if(e.type==='model_detected')m.observed=true;if(isPick)m.picks++;if(e.type==='success')m.success++;if(e.type==='rate_limited')m.rate++;if(e.type==='network_retryable')m.retry++;if(e.type==='auth_failed')m.auth++;if(e.type==='transient_status')m.transient++;s.models.set(e.model,m);}else if(isPick||isOutcome){s.unscoped=s.unscoped||{picks:0,success:0,rate:0,retry:0,auth:0,transient:0};if(isPick)s.unscoped.picks++;if(e.type==='success')s.unscoped.success++;if(e.type==='rate_limited')s.unscoped.rate++;if(e.type==='network_retryable')s.unscoped.retry++;if(e.type==='auth_failed')s.unscoped.auth++;if(e.type==='transient_status')s.unscoped.transient++;}}return stats;}
+function emptyStat(provider,slot,total,key){return{provider,slot,total,key,picks:0,success:0,rate:0,retry:0,auth:0,transient:0,last:'',lastPick:'',models:new Map()};}
+function buildStats(){const stats=new Map();for(const p of state.providers){for(const k of (p.keys||[]))stats.set(keyId(p.name,k.slot,k.key),emptyStat(p.name,k.slot,k.total,k.key));}for(const e of state.events){if(!isKeyEvent(e))continue;const id=keyId(e.provider,e.slot,e.key);if(!stats.has(id))stats.set(id,emptyStat(e.provider,e.slot,e.total,e.key||'***'));const s=stats.get(id);const isPick=PICK_TYPES.includes(e.type);const isOutcome=['success','rate_limited','network_retryable','auth_failed','transient_status','inflight_timeout'].includes(e.type);if(isPick){s.picks++;s.lastPick=e.ts||s.lastPick;}if(e.type==='success')s.success++;if(e.type==='rate_limited')s.rate++;if(e.type==='network_retryable')s.retry++;if(e.type==='auth_failed')s.auth++;if(e.type==='transient_status'||e.type==='inflight_timeout')s.transient++;s.last=e.ts||s.last;if(e.model){let m=s.models.get(e.model);if(!m)m={picks:0,success:0,rate:0,retry:0,auth:0,transient:0,last:'',lastPick:'',observed:false};if(e.type==='model_detected')m.observed=true;if(isPick){m.picks++;m.lastPick=e.ts||m.lastPick;}if(e.type==='success')m.success++;if(e.type==='rate_limited')m.rate++;if(e.type==='network_retryable')m.retry++;if(e.type==='auth_failed')m.auth++;if(e.type==='transient_status'||e.type==='inflight_timeout')m.transient++;m.last=e.ts||m.last;s.models.set(e.model,m);}else if(isPick||isOutcome){s.unscoped=s.unscoped||{picks:0,success:0,rate:0,retry:0,auth:0,transient:0,last:'',lastPick:''};if(isPick){s.unscoped.picks++;s.unscoped.lastPick=e.ts||s.unscoped.lastPick;}if(e.type==='success')s.unscoped.success++;if(e.type==='rate_limited')s.unscoped.rate++;if(e.type==='network_retryable')s.unscoped.retry++;if(e.type==='auth_failed')s.unscoped.auth++;if(e.type==='transient_status'||e.type==='inflight_timeout')s.unscoped.transient++;s.unscoped.last=e.ts||s.unscoped.last;}}const now=Date.now();for(const s of stats.values()){ageStalePending(s,now);for(const m of s.models.values())ageStalePending(m,now);if(s.unscoped)ageStalePending(s.unscoped,now);}return stats;}
+function ageStalePending(v,now){const pending=pendingCount(v);if(!pending)return;const ts=Date.parse(v.lastPick||v.last||'');if(Number.isFinite(ts)&&now-ts>STALE_PENDING_MS){v.transient=(v.transient||0)+pending;v.stalePending=(v.stalePending||0)+pending;}}
 function providerRows(provider,stats){return (provider?.keys||[]).map(k=>stats.get(keyId(provider.name,k.slot,k.key))||emptyStat(provider.name,k.slot,k.total,k.key));}
 function pendingCount(v){return Math.max(0,(v.picks||0)-((v.success||0)+(v.rate||0)+(v.retry||0)+(v.transient||0)+(v.auth||0)));}
 function modelChips(row){const entries=[...row.models.entries()];const un=row.unscoped||{picks:0,success:0,rate:0,retry:0,auth:0,transient:0};const unPending=pendingCount(un);if(!entries.length){const total=row.picks+row.success+row.rate+row.retry+row.transient+row.auth;return total?`<span class="model-chip">unscoped · p:${row.picks} ok:${row.success} pending:${pendingCount(row)} rl:${row.rate} retry:${row.retry+row.transient}</span>`:'<span class="model-chip">no model events yet</span>';}const chips=entries.map(([m,v])=>{const onlyObserved=v.observed&&!v.picks&&!v.success&&!v.rate&&!v.retry&&!v.transient&&!v.auth;return `<span class="model-chip">${esc(m)} · ${onlyObserved?'observed':`p:${v.picks} ok:${v.success} pending:${pendingCount(v)} rl:${v.rate} retry:${v.retry+v.transient}`}</span>`;});if(un.picks||un.success||un.rate||un.retry||un.transient||un.auth)chips.push(`<span class="model-chip">unscoped totals · p:${un.picks} ok:${un.success} pending:${unPending} rl:${un.rate} retry:${un.retry+un.transient}</span>`);return chips.join('');}

multi-provider-key-rotator.cjs

@@ -112,7 +112,7 @@ const EVENT_LOG_FILE = process.env.KEY_ROTATOR_EVENT_LOG_FILE || '/tmp/huggingcl
 const EVENT_LOG_MAX_BYTES = Math.max(64 * 1024, parseInt(process.env.KEY_ROTATOR_EVENT_LOG_MAX_BYTES || '', 10) || 1024 * 1024);
 const INFLIGHT_TTL_MS = Math.max(
   30_000,
-  Math.min(30 * 60_000, parseInt(process.env.KEY_INFLIGHT_TTL_MS || '', 10) || 5 * 60_000),
+  Math.min(30 * 60_000, parseInt(process.env.KEY_INFLIGHT_TTL_MS || '', 10) || 30_000),
 );
 const REQUEST_MODEL_SNIFF_MAX_BYTES = Math.max(
   16 * 1024,
@@ -141,6 +141,7 @@ const STICKY_PROVIDER_SET = new Set(
     .map(s => s.trim().toLowerCase())
     .filter(Boolean),
 );
+const UNKNOWN_MODEL_SCOPE = '__unknown_model__';
 
 // Maximum ms to respect from a Retry-After header.
 // Old cap was 10s — too low for Gemini/Google which often returns 60s+.
@@ -440,17 +441,27 @@ function isGeminiOpenAICompatPath(pathOrUrl) {
  */
 function getKeyExpiry(p, key, model) {
   let expiry = p.keyState.get(key)?.blacklistedUntil ?? 0;
-  if (p.modelKeyState && model) {
-    const mks = p.modelKeyState.get(`${key}:${model}`);
+  if (p.modelKeyState) {
+    const mks = p.modelKeyState.get(`${key}:${scopedModelKey(model)}`);
     if (mks && mks.blacklistedUntil > expiry) expiry = mks.blacklistedUntil;
   }
   return expiry;
 }
 
 function stickyBucketForProvider(p, model) {
-  const rawScope = String(process.env.KEY_STICKY_SCOPE || 'provider').trim().toLowerCase();
-  const scope = rawScope === 'model' || rawScope === 'per-model' ? 'model' : 'provider';
-  return scope === 'provider' ? '__provider__' : (model || '__default__');
+  const rawScope = String(process.env.KEY_STICKY_SCOPE || '').trim().toLowerCase();
+  const scope = rawScope === 'provider'
+    ? 'provider'
+    : rawScope === 'model' || rawScope === 'per-model'
+      ? 'model'
+      : p?.perModelLimits
+        ? 'model'
+        : 'provider';
+  return scope === 'provider' ? '__provider__' : (model || UNKNOWN_MODEL_SCOPE);
+}
+
+function scopedModelKey(model) {
+  return model || UNKNOWN_MODEL_SCOPE;
 }
 
 function isStickyProvider(p) {
@@ -464,14 +475,17 @@ function rememberStickyKey(p, model, key) {
 
 function clearStickyKey(p, key, model) {
   if (!p?.stickyKeys || !key) return;
-  if (model) {
+  const hasScopedModelArg = arguments.length >= 3;
+  if (hasScopedModelArg) {
     const bucket = stickyBucketForProvider(p, model);
     if (p.stickyKeys.get(bucket) === key) p.stickyKeys.delete(bucket);
     // Also clear the ambiguous fallback bucket if this key was selected before
     // a Gemini OpenAI-compatible request body revealed its model.  Do not clear
     // other model buckets: Gemini quota failures are model-scoped.
-    const fallbackBucket = stickyBucketForProvider(p, null);
-    if (p.stickyKeys.get(fallbackBucket) === key) p.stickyKeys.delete(fallbackBucket);
+    if (model) {
+      const fallbackBucket = stickyBucketForProvider(p, null);
+      if (p.stickyKeys.get(fallbackBucket) === key) p.stickyKeys.delete(fallbackBucket);
+    }
     return;
   }
   for (const [bucket, stickyKey] of p.stickyKeys) {
@@ -632,14 +646,15 @@ function isActive(p, key, model) {
   }
 
   // ── Per-model check (gemini etc.) ──────────────────────────────────────────
-  if (p.modelKeyState && model) {
-    const mKey = `${key}:${model}`;
+  if (p.modelKeyState) {
+    const scopedModel = scopedModelKey(model);
+    const mKey = `${key}:${scopedModel}`;
     const mks  = p.modelKeyState.get(mKey);
     if (mks && mks.blacklistedUntil !== 0) {
-      if (Date.now() < mks.blacklistedUntil) return false;   // blocked for this model
+      if (Date.now() < mks.blacklistedUntil) return false;   // blocked for this model/unknown-model scope
       mks.blacklistedUntil = 0;
       if (mks.strikes > 0) mks.strikes -= 1;
-      debug(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} back in pool for model=${model} (strikes now ${mks.strikes})`);
+      debug(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} back in pool for model=${model || 'unknown'} (strikes now ${mks.strikes})`);
     }
   }
 
@@ -668,8 +683,9 @@ function recordFailure(p, key, model, retryAfterMs) {
   // our exponential cooldown.  This prevents hammering a key before its quota resets.
   const serverHintMs = (typeof retryAfterMs === 'number' && retryAfterMs > 0) ? retryAfterMs : 0;
 
-  if (p.modelKeyState && model) {
-    const mKey = `${key}:${model}`;
+  if (p.modelKeyState) {
+    const scopedModel = scopedModelKey(model);
+    const mKey = `${key}:${scopedModel}`;
     let mks = p.modelKeyState.get(mKey);
     if (!mks) { mks = makeKeyState(); p.modelKeyState.set(mKey, mks); }
 
@@ -689,9 +705,9 @@ function recordFailure(p, key, model, retryAfterMs) {
     // ★ Set blacklistedUntil FIRST so it is always written even if the log below throws.
     mks.blacklistedUntil = Math.max(mks.blacklistedUntil || 0, Date.now() + cooldown);
     if (isPerm)
-      warn(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} model=${model} hit ${MAX_STRIKES} strikes — suspended for ${formatHours(PERM_SUSPEND_MS)}h (quota likely exhausted for this model)`);
+      warn(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} model=${model || 'unknown'} hit ${MAX_STRIKES} strikes — suspended for ${formatHours(PERM_SUSPEND_MS)}h (quota likely exhausted for this model)`);
     else
-      debug(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} model=${model} strike ${mks.strikes}/${MAX_STRIKES} — backoff ${Math.round(cooldown / 1000)}s${serverHintMs > 0 ? ` (server-hint ${Math.round(serverHintMs/1000)}s)` : ''}`);
+      debug(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} model=${model || 'unknown'} strike ${mks.strikes}/${MAX_STRIKES} — backoff ${Math.round(cooldown / 1000)}s${serverHintMs > 0 ? ` (server-hint ${Math.round(serverHintMs/1000)}s)` : ''}`);
     return;
   }
 
@@ -725,15 +741,18 @@ function recordFailure(p, key, model, retryAfterMs) {
  * Called on transient retryable failures (non-quota/rate):
  * applies short cooldown without incrementing strikes.
  */
-function recordTransientFailure(p, key) {
-  let ks = p.keyState.get(key);
-  if (!ks) { ks = makeKeyState(); p.keyState.set(key, ks); }
+function recordTransientFailure(p, key, model = null) {
+  if (!p || !key) return;
+  const stateMap = p.modelKeyState ? p.modelKeyState : p.keyState;
+  const stateKey = p.modelKeyState ? `${key}:${scopedModelKey(model)}` : key;
+  let ks = stateMap.get(stateKey);
+  if (!ks) { ks = makeKeyState(); stateMap.set(stateKey, ks); }
   ks.lastFailureAt = Date.now();
   const jitter = 1 + ((Math.random() * 2 - 1) * (COOLDOWN_JITTER_PCT / 100));
   const cooldown = Math.max(1000, Math.round(BASE_COOLDOWN_MS * jitter));
   ks.blacklistedUntil = Math.max(ks.blacklistedUntil || 0, Date.now() + cooldown);
   const secs = Math.round(cooldown / 1000);
-  debug(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} transient backoff ${secs}s (strikes unchanged)`);
+  debug(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} transient backoff ${secs}s${p.modelKeyState ? ` model=${model || 'unknown'}` : ''} (strikes unchanged)`);
 }
 
 /**
@@ -754,15 +773,17 @@ function recordSuccess(p, key, model) {
     }
   }
 
-  // Also clear model-specific state on success
-  if (p.modelKeyState && model) {
-    const mKey = `${key}:${model}`;
+  // Also clear model-specific state on success.  If model is still unknown,
+  // clear only the unknown-model scope; never clear other Gemini model buckets.
+  if (p.modelKeyState) {
+    const scopedModel = scopedModelKey(model);
+    const mKey = `${key}:${scopedModel}`;
     const mks  = p.modelKeyState.get(mKey);
     if (mks && (mks.strikes > 0 || mks.blacklistedUntil > 0)) {
       mks.strikes = 0;
       mks.lastFailureAt = 0;
       mks.blacklistedUntil = 0;
-      debug(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} model=${model} recovered — strikes reset`);
+      debug(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} model=${model || 'unknown'} recovered — strikes reset`);
     }
   }
 }
@@ -802,9 +823,9 @@ function removeInFlightToken(p, key, token) {
   else p.inFlightTimers.delete(key);
 }
 
-function beginInFlight(p, key) {
+function beginInFlight(p, key, model = null) {
   if (!p || !key) return null;
-  const token = { done: false, timer: null };
+  const token = { done: false, timer: null, model: model || null };
   p.inFlight.set(key, (p.inFlight.get(key) || 0) + 1);
   token.timer = setTimeout(() => {
     if (token.done) return;
@@ -813,7 +834,14 @@ function beginInFlight(p, key) {
     const before = p.inFlight.get(key) || 0;
     if (before > 0) {
       const next = decrementInFlight(p, key);
-      emitEvent('inflight_timeout', p, key, { inflightBefore: before, inflightAfter: next, ttlMs: INFLIGHT_TTL_MS });
+      // A timeout here means the rotator saw a key pick but no provider headers,
+      // completion, or transport error before the TTL.  For OpenClaw failover
+      // paths this otherwise leaves the sticky bucket pinned forever and the
+      // dashboard shows only "pending/no response observed".  Treat it as a
+      // transient key failure so the next request can rotate to another key.
+      try { recordTransientFailure(p, key, token.model || null); } catch (_) {}
+      try { clearStickyKey(p, key, token.model || null); } catch (_) {}
+      emitEvent('inflight_timeout', p, key, { model: token.model || null, inflightBefore: before, inflightAfter: next, ttlMs: INFLIGHT_TTL_MS, classifiedAs: 'transient' });
     }
   }, INFLIGHT_TTL_MS);
   token.timer.unref?.();
@@ -874,16 +902,25 @@ function nextKey(p, model) {
     const stickyKey = p.stickyKeys.get(stickyBucketForProvider(p, model));
     if (stickyKey && p.keys.includes(stickyKey) && isActive(p, stickyKey, model)) {
       const inflight = p.inFlight.get(stickyKey) || 0;
-      verbosePickLog(`[key-rotator] ${p.name}: sticky picked ${keySlot(p, stickyKey)}${keyMask(stickyKey)}${model ? ` model=${model}` : ''} inflight=${inflight + 1}/${MAX_INFLIGHT_PER_KEY}`);
-      if (inflight >= MAX_INFLIGHT_PER_KEY) {
-        warn(`[key-rotator] ${p.name}: sticky key saturated, still reusing ${keySlot(p, stickyKey)}${keyMask(stickyKey)}${model ? ` model=${model}` : ''} inflight=${inflight + 1}/${MAX_INFLIGHT_PER_KEY} until it fails/exhausts`);
-        emitEvent('sticky_saturated_reuse', p, stickyKey, { model, inflight: inflight + 1, maxInflight: MAX_INFLIGHT_PER_KEY });
-      } else {
+      if (inflight < MAX_INFLIGHT_PER_KEY) {
+        verbosePickLog(`[key-rotator] ${p.name}: sticky picked ${keySlot(p, stickyKey)}${keyMask(stickyKey)}${model ? ` model=${model}` : ''} inflight=${inflight + 1}/${MAX_INFLIGHT_PER_KEY}`);
         emitEvent('sticky_pick', p, stickyKey, { model, inflight: inflight + 1, maxInflight: MAX_INFLIGHT_PER_KEY });
+        return { key: stickyKey, waitMs: 0 };
       }
-      return { key: stickyKey, waitMs: 0 };
+
+      // Do not keep piling requests onto one sticky key when OpenClaw has not
+      // produced provider headers/completion/error for previous picks.  That was
+      // the real cause of dashboards like "pick 14, pending 14, rate 0": sticky
+      // mode intentionally reused the same key even after it was saturated.  Clear
+      // the bucket and let the normal active-key scan below choose another key;
+      // only if every key is saturated will the fallback path reuse the least-bad
+      // candidate.
+      warn(`[key-rotator] ${p.name}: sticky key saturated, rotating away from ${keySlot(p, stickyKey)}${keyMask(stickyKey)}${model ? ` model=${model}` : ''} inflight=${inflight}/${MAX_INFLIGHT_PER_KEY}`);
+      emitEvent('sticky_saturated_rotate', p, stickyKey, { model, inflight, maxInflight: MAX_INFLIGHT_PER_KEY });
+      clearStickyKey(p, stickyKey, model);
+    } else if (stickyKey) {
+      clearStickyKey(p, stickyKey, model);
     }
-    if (stickyKey) clearStickyKey(p, stickyKey, model);
   }
 
   let bestPick = null;
@@ -907,7 +944,7 @@ function nextKey(p, model) {
       // Score: prefer keys with fewer recent failures and lower in-flight count.
       // For perModelLimits, also factor in model-specific strike count.
       const ks  = p.keyState.get(key) || makeKeyState();
-      const mks = (p.modelKeyState && model) ? (p.modelKeyState.get(`${key}:${model}`) || makeKeyState()) : makeKeyState();
+      const mks = p.modelKeyState ? (p.modelKeyState.get(`${key}:${scopedModelKey(model)}`) || makeKeyState()) : makeKeyState();
       const recentFailPenalty =
         (ks.lastFailureAt  > 0 && (Date.now() - ks.lastFailureAt)  < FAILURE_DECAY_MS ? 100 : 0) +
         (mks.lastFailureAt > 0 && (Date.now() - mks.lastFailureAt) < FAILURE_DECAY_MS ? 100 : 0);
@@ -1047,9 +1084,11 @@ function handleStatus(p, key, status, model, retryAfterMs, errorInfo) {
   }
 
   if (failureKind === 'transient') {
-    // Transient server errors are not model-specific — penalise key globally.
-    recordTransientFailure(p, key);
-    clearStickyKey(p, key);
+    // For per-model providers, keep transient cooldowns scoped to the current
+    // model/unknown-model bucket so one Gemini model does not suppress the key
+    // for all other Gemini models.
+    recordTransientFailure(p, key, model);
+    clearStickyKey(p, key, model);
     warn(`[key-rotator] ${p.name}: transient status=${status} on ${keySlot(p, key)}${keyMask(key)}`);
     emitEvent('transient_status', p, key, { status, model, ...errorFields });
     return;
@@ -1061,7 +1100,7 @@ function handleStatus(p, key, status, model, retryAfterMs, errorInfo) {
   }
 }
 
-function handleTransportError(p, key, err) {
+function handleTransportError(p, key, err, model = null) {
   if (!p || !key) return;
   // Node.js 18+ undici fetch throws TypeError: "fetch failed" where the actual
   // network error code lives in err.cause.code (e.g. ECONNRESET, ETIMEDOUT,
@@ -1071,12 +1110,22 @@ function handleTransportError(p, key, err) {
     ? String(err.code || err.cause?.code).toUpperCase()
     : '';
   const name = String(err?.name || '');
+  const message = String(err?.message || err?.cause?.message || '');
+  const haystack = `${name} ${message}`.toLowerCase();
+  const looksRateOrQuota = /rate.?limit|too.?many|quota|resource.?exhaust|usage.?limit|insufficient.?quota|capacity.?exceeded|tokens?.?per|requests?.?per|rate_limit|rate.?limited|userratelimit|dailylimit|limitexceeded/.test(haystack);
+  if (looksRateOrQuota) {
+    recordFailure(p, key, model, 0);
+    clearStickyKey(p, key, model);
+    warn(`[key-rotator] ${p.name}: transport/failover quota signal ${name || 'Error'}${code ? ` code=${code}` : ''} on ${keySlot(p, key)}${keyMask(key)}${model ? ` model=${model}` : ''}`);
+    emitEvent('rate_limited', p, key, { model, name: name || 'Error', code, source: 'transport_error', message: message.slice(0, 240) });
+    return;
+  }
   const retryable = classifyRetryableFailure(undefined, code) || name === 'AbortError';
   if (retryable) {
-    recordTransientFailure(p, key);
-    clearStickyKey(p, key);
-    warn(`[key-rotator] ${p.name}: retryable network ${name || 'Error'}${code ? ` code=${code}` : ''} on ${keySlot(p, key)}${keyMask(key)}`);
-    emitEvent('network_retryable', p, key, { name: name || 'Error', code });
+    recordTransientFailure(p, key, model);
+    clearStickyKey(p, key, model);
+    warn(`[key-rotator] ${p.name}: retryable network ${name || 'Error'}${code ? ` code=${code}` : ''} on ${keySlot(p, key)}${keyMask(key)}${model ? ` model=${model}` : ''}`);
+    emitEvent('network_retryable', p, key, { model, name: name || 'Error', code });
   }
 }
 
@@ -1614,7 +1663,7 @@ function wrapUndiciHandler(handler, provider, key, inFlightToken, getModel) {
           } finally {
             // User handlers may throw/rethrow; the rotator still owns the
             // in-flight token and transport error classification for this key.
-            settle(() => { if (!statusHandled) { try { handleTransportError(provider, key, err); } catch (_) {} } });
+            settle(() => { if (!statusHandled) { try { handleTransportError(provider, key, err, currentModel()); } catch (_) {} } });
           }
         };
       }
@@ -1680,7 +1729,7 @@ function patchUndiciDispatch(proto, tag) {
 
         if (key) {
           usedKey = key; usedProvider = provider; usedModel = model;
-          usedInFlight = beginInFlight(usedProvider, usedKey);
+          usedInFlight = beginInFlight(usedProvider, usedKey, usedModel);
 
           const newOptions = { ...options };
 
@@ -1709,7 +1758,7 @@ function patchUndiciDispatch(proto, tag) {
             usedProvider,
             usedKey,
             () => usedModel,
-            (model) => { usedModel = model; },
+            (model) => { usedModel = model; if (usedInFlight) usedInFlight.model = model; },
           );
           const wrappedHandler = wrapUndiciHandler(handler, usedProvider, usedKey, usedInFlight, () => usedModel);
           return runInRotatorRequest(() => origDispatch.call(this, newOptions, wrappedHandler));
@@ -1892,7 +1941,7 @@ function patchFetch() {
           if (key) {
             triedKeys.add(key);
             usedKey = key;
-            usedInFlight = beginInFlight(provider, key);
+            usedInFlight = beginInFlight(provider, key, model);
           }
 
           const attemptArgs = buildAttemptFetchArgs(input, init, provider, usedKey);
@@ -1923,7 +1972,7 @@ function patchFetch() {
         } catch (err) {
           lastErr = err;
           try { endInFlight(provider, usedKey, usedInFlight); } catch (_) {}
-          try { handleTransportError(provider, usedKey, err); } catch (_) {}
+          try { handleTransportError(provider, usedKey, err, model); } catch (_) {}
           // Node.js 18+ undici fetch: network errors are TypeError("fetch failed")
           // where the real code (ECONNRESET, ETIMEDOUT, ENOTFOUND …) is in
           // err.cause.code.  Check that first before falling back to err.code.
@@ -1989,7 +2038,7 @@ function patchHttpModule(mod) {
 
         if (key) {
           usedKey = key; usedProvider = provider; usedModel = model;
-          usedInFlight = beginInFlight(usedProvider, usedKey);
+          usedInFlight = beginInFlight(usedProvider, usedKey, usedModel);
           if (provider.queryParam) {
             const hasOptionsArg = args[1] && typeof args[1] === 'object' && typeof args[1].on !== 'function';
             const u = new URL(String(
@@ -2081,6 +2130,7 @@ function patchHttpModule(mod) {
                   const bodyModel = extractModelFromBody(fullBody);
                   if (bodyModel) {
                     usedModel = bodyModel;
+                    if (usedInFlight) usedInFlight.model = usedModel;
                     promoteStickyKeyModel(usedProvider, usedKey, null, usedModel);
                     emitEvent('model_detected', usedProvider, usedKey, { model: usedModel, source: 'http_request_body' });
                     debug(`[key-rotator] ${usedProvider.name}: (http) model extracted from request body: ${usedModel}`);
@@ -2149,7 +2199,7 @@ function patchHttpModule(mod) {
       req.on('error', (err) => {
         try { endInFlight(usedProvider, usedKey, usedInFlight); } catch (_) {}
         if (!statusHandled) {
-          try { handleTransportError(usedProvider, usedKey, err); } catch (_) {}
+          try { handleTransportError(usedProvider, usedKey, err, usedModel); } catch (_) {}
         }
       });
     }
@@ -2181,7 +2231,7 @@ if (hasProviderKeys) {
   patchUndici();         // covers OpenClaw gateway's bundled undici AI calls
   startDiagnostics();
 
-  debug(`[key-rotator] loaded — cooldown base:${BASE_COOLDOWN_MS/1000}s max-strikes:${MAX_STRIKES} perm-suspend:${formatHours(PERM_SUSPEND_MS)}h (cap 16h) max-inflight-per-key:${MAX_INFLIGHT_PER_KEY} max-retry-after:${MAX_RETRY_AFTER_MS/1000}s max-key-wait:${MAX_KEY_WAIT_MS/1000}s diagnostics:${DIAGNOSTICS_ENABLED ? 'on' : 'off'} log-level:${LOG_LEVEL} verbose-picks:${VERBOSE_PICKS ? 'on' : 'off'} suspended-last-resort:${USE_SUSPENDED_KEY_AS_LAST_RESORT ? 'on' : 'off'} per-model-providers:${providerState.filter(p => p.perModelLimits).map(p => p.name).join(',') || 'none'} model-from-body:on model-sniff-max:${REQUEST_MODEL_SNIFF_MAX_BYTES} error-sniff-max:${ERROR_BODY_SNIFF_MAX_BYTES} inflight-ttl:${INFLIGHT_TTL_MS}ms sticky-until-failure:${STICKY_UNTIL_FAILURE ? 'on' : 'off'} sticky-scope:${String(process.env.KEY_STICKY_SCOPE || 'provider').trim().toLowerCase() || 'provider'} sticky-providers:${[...STICKY_PROVIDER_SET].join(',') || 'none'} llm-fallback-providers:${LLM_FALLBACK_PROVIDER_SET ? [...LLM_FALLBACK_PROVIDER_SET].join(',') : 'all'}`);
+  debug(`[key-rotator] loaded — cooldown base:${BASE_COOLDOWN_MS/1000}s max-strikes:${MAX_STRIKES} perm-suspend:${formatHours(PERM_SUSPEND_MS)}h (cap 16h) max-inflight-per-key:${MAX_INFLIGHT_PER_KEY} max-retry-after:${MAX_RETRY_AFTER_MS/1000}s max-key-wait:${MAX_KEY_WAIT_MS/1000}s diagnostics:${DIAGNOSTICS_ENABLED ? 'on' : 'off'} log-level:${LOG_LEVEL} verbose-picks:${VERBOSE_PICKS ? 'on' : 'off'} suspended-last-resort:${USE_SUSPENDED_KEY_AS_LAST_RESORT ? 'on' : 'off'} per-model-providers:${providerState.filter(p => p.perModelLimits).map(p => p.name).join(',') || 'none'} model-from-body:on model-sniff-max:${REQUEST_MODEL_SNIFF_MAX_BYTES} error-sniff-max:${ERROR_BODY_SNIFF_MAX_BYTES} inflight-ttl:${INFLIGHT_TTL_MS}ms sticky-until-failure:${STICKY_UNTIL_FAILURE ? 'on' : 'off'} sticky-scope:${String(process.env.KEY_STICKY_SCOPE || 'auto').trim().toLowerCase() || 'auto'} sticky-providers:${[...STICKY_PROVIDER_SET].join(',') || 'none'} llm-fallback-providers:${LLM_FALLBACK_PROVIDER_SET ? [...LLM_FALLBACK_PROVIDER_SET].join(',') : 'all'}`);
   emitEvent('rotator_loaded', null, null, {
     providers: providerState.filter(p => p.keys.length).map(p => ({ name: p.name, total: p.keys.length })),
     logLevel: LOG_LEVEL,
@@ -2190,7 +2240,7 @@ if (hasProviderKeys) {
     modelSniffMaxBytes: REQUEST_MODEL_SNIFF_MAX_BYTES,
     errorBodySniffMaxBytes: ERROR_BODY_SNIFF_MAX_BYTES,
     stickyUntilFailure: STICKY_UNTIL_FAILURE,
-    stickyScope: String(process.env.KEY_STICKY_SCOPE || 'provider').trim().toLowerCase() || 'provider',
+    stickyScope: String(process.env.KEY_STICKY_SCOPE || 'auto').trim().toLowerCase() || 'auto',
     stickyProviders: [...STICKY_PROVIDER_SET],
     llmFallbackProviders: LLM_FALLBACK_PROVIDER_SET ? [...LLM_FALLBACK_PROVIDER_SET] : ['*'],
   });