Fix custom env builder card layout

env-builder.html

@@ -788,11 +788,20 @@ body {
   line-height: 1.45;
 }
 
+#customRows.cards {
+  grid-template-columns: 1fr;
+}
+
+.custom-env-card {
+  width: 100%;
+  max-width: none;
+}
+
 .custom-env-card .card-top { margin-bottom: 12px; }
 
 .custom-card-grid {
   display: grid;
-  grid-template-columns: minmax(180px, 1fr) minmax(220px, 1.4fr) minmax(130px, .7fr) auto;
+  grid-template-columns: minmax(160px, 1fr) minmax(200px, 1.4fr) minmax(120px, .65fr) max-content;
   gap: 8px;
   align-items: end;
 }
@@ -840,7 +849,8 @@ body {
   color: var(--red);
 }
 
-@media (max-width: 900px) {
+@media (max-width: 720px) {
+  .custom-env-toolbar { align-items: stretch; flex-direction: column; }
   .custom-card-grid { grid-template-columns: 1fr; }
   .custom-remove { width: 100%; }
 }

env-builder.js

@@ -2587,7 +2587,21 @@ function addCustomRow(key = '', val = '', enabled = false, tag = 'optional') {
     updateCounts();
   });
   row.querySelector(`[data-custom-remove="${id}"]`)?.addEventListener('click', () => {
-    row.remove();
+    const remaining = document.querySelectorAll('[data-custom-row]').length;
+    if (remaining <= 1) {
+      const keyInput = row.querySelector(`[data-ck="${id}"]`);
+      const valueInput = row.querySelector(`[data-cv="${id}"]`);
+      const tagInput = row.querySelector(`[data-ct="${id}"]`);
+      if (keyInput) keyInput.value = '';
+      if (valueInput) valueInput.value = '';
+      if (tagInput) tagInput.value = 'optional';
+      if (enabledInput) enabledInput.checked = false;
+      row.dataset.enabled = '0';
+      row.classList.remove('selected');
+      updateCustomRowMeta(row);
+    } else {
+      row.remove();
+    }
     refresh();
     filter();
   });

key-rotator-manager.html

@@ -5,7 +5,7 @@
   <meta name="viewport" content="width=device-width,initial-scale=1" />
   <title>HuggingClaw · API Key Rotator</title>
   <style>
-    :root{color-scheme:dark;--bg:#070711;--panel:#111120;--panel2:#17172a;--line:#292945;--text:#f8f7ff;--muted:#9892b8;--soft:#c7c2e6;--good:#22c55e;--warn:#f5c542;--bad:#fb7185;--blue:#60a5fa;--violet:#a78bfa}*{box-sizing:border-box}body{margin:0;min-height:100vh;background:radial-gradient(circle at top left,#25145a 0,#070711 34%,#070711 100%);font-family:Inter,ui-sans-serif,system-ui,-apple-system,sans-serif;color:var(--text);font-size:13px}main{width:min(1280px,calc(100% - 28px));margin:0 auto;padding:28px 0 44px}.top{display:flex;align-items:flex-start;justify-content:space-between;gap:18px;margin-bottom:18px}.eyebrow{font-size:.7rem;letter-spacing:.18em;text-transform:uppercase;color:var(--muted);font-weight:900}h1{margin:6px 0 8px;font-size:clamp(1.6rem,4vw,2.7rem);line-height:1}.sub{color:var(--soft);max-width:860px;line-height:1.55}.actions{display:flex;gap:10px;flex-wrap:wrap}.btn{border:1px solid var(--line);background:rgba(255,255,255,.06);color:var(--text);border-radius:11px;padding:10px 14px;text-decoration:none;font-weight:850;cursor:pointer}.btn.primary{background:#fff;color:#050510}.btn:hover{filter:brightness(1.08)}.grid{display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:12px;margin:18px 0}.card{background:linear-gradient(180deg,rgba(255,255,255,.055),rgba(255,255,255,.025));border:1px solid var(--line);border-radius:18px;padding:16px;box-shadow:0 18px 45px rgba(0,0,0,.22)}.metric-title{color:var(--muted);text-transform:uppercase;letter-spacing:.16em;font-size:.66rem;font-weight:900}.metric-value{font-size:1.65rem;font-weight:950;margin-top:8px}.metric-detail{color:var(--muted);margin-top:6px;line-height:1.45}.ok{color:var(--good)}.warn{color:var(--warn)}.bad{color:var(--bad)}.blue{color:var(--blue)}.layout{display:grid;grid-template-columns:340px minmax(0,1fr);gap:14px}.panel-title{display:flex;justify-content:space-between;align-items:center;gap:10px;margin-bottom:12px}.panel-title h2{font-size:1rem;margin:0}.pill{display:inline-flex;align-items:center;gap:6px;border:1px solid var(--line);border-radius:999px;padding:5px 9px;color:var(--soft);background:rgba(255,255,255,.035);font-size:.72rem;font-weight:850}.dot{width:7px;height:7px;border-radius:50%;background:var(--muted)}.dot.live{background:var(--good);box-shadow:0 0 15px var(--good)}.providers{display:flex;flex-direction:column;gap:9px}.provider{border:1px solid var(--line);border-radius:14px;background:rgba(0,0,0,.12);padding:12px;cursor:pointer}.provider.active{border-color:var(--blue);box-shadow:0 0 0 1px rgba(96,165,250,.25)}.provider-top{display:flex;justify-content:space-between;align-items:center;gap:12px}.provider-name{font-weight:950}.provider-meta{color:var(--muted);font-size:.78rem;margin-top:5px}.bar{height:8px;background:#22223a;border-radius:999px;overflow:hidden;margin-top:10px}.bar>span{display:block;height:100%;background:linear-gradient(90deg,var(--violet),var(--blue));border-radius:999px}.toolbar{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px}.input{flex:1;min-width:180px;border:1px solid var(--line);background:#0c0c18;color:var(--text);border-radius:11px;padding:10px 12px;outline:none}.select{border:1px solid var(--line);background:#0c0c18;color:var(--text);border-radius:11px;padding:10px 12px}.toggle{display:flex;align-items:center;gap:7px;color:var(--soft);font-weight:800}.events{display:flex;flex-direction:column;gap:9px;max-height:620px;overflow:auto;padding-right:4px}.event{border:1px solid var(--line);border-left-width:4px;border-radius:14px;background:rgba(0,0,0,.16);padding:12px;display:grid;grid-template-columns:155px minmax(0,1fr) auto;gap:12px}.event.pick,.event.sticky_pick,.event.pick_retry_fresh,.event.model_detected{border-left-color:var(--blue)}.event.success{border-left-color:var(--good)}.event.rate_limited,.event.auth_failed,.event.transport_aborted,.event.all_suspended_pick,.event.all_suspended_withheld{border-left-color:var(--warn)}.event.network_retryable,.event.transient_status,.event.saturated_reuse,.event.sticky_saturated_reuse,.event.sticky_saturated_rotate,.event.inflight_timeout{border-left-color:var(--bad)}.time{color:var(--muted);font-variant-numeric:tabular-nums}.etype{font-weight:950}.msg{color:var(--soft);line-height:1.45;word-break:break-word}.key,.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:#e9d5ff}.empty{padding:34px;text-align:center;color:var(--muted);border:1px dashed var(--line);border-radius:16px}.foot{color:var(--muted);margin-top:16px;line-height:1.55}.kbd{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;background:#23233a;border:1px solid #363653;border-radius:7px;padding:2px 6px;color:var(--text)}.detail-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:10px;margin-bottom:12px}.mini{border:1px solid var(--line);border-radius:13px;padding:11px;background:rgba(0,0,0,.12)}.mini b{display:block;font-size:1.15rem;margin-top:4px}.key-table-wrap{overflow:auto}.key-table{width:100%;border-collapse:separate;border-spacing:0 8px;min-width:780px}.key-table th{color:var(--muted);font-size:.68rem;text-align:left;text-transform:uppercase;letter-spacing:.12em}.key-table td{background:rgba(0,0,0,.15);border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:10px;vertical-align:top}.key-table td:first-child{border-left:1px solid var(--line);border-radius:12px 0 0 12px}.key-table td:last-child{border-right:1px solid var(--line);border-radius:0 12px 12px 0}.status{font-weight:900}.status.used{color:var(--good)}.status.unused{color:var(--muted)}.model-chip{display:inline-flex;margin:2px 4px 2px 0;padding:4px 7px;border:1px solid var(--line);border-radius:999px;color:var(--soft);background:rgba(255,255,255,.035);font-size:.72rem}.section{margin-top:14px}.session-note{border:1px solid rgba(96,165,250,.28);background:rgba(96,165,250,.08);border-radius:14px;padding:12px;color:var(--soft);margin-bottom:12px}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}@media(max-width:1050px){.grid{grid-template-columns:repeat(2,minmax(0,1fr))}.detail-grid{grid-template-columns:repeat(2,minmax(0,1fr))}.layout{grid-template-columns:1fr}.top{flex-direction:column}.event{grid-template-columns:1fr}.events{max-height:none}}@media(max-width:620px){.grid,.detail-grid{grid-template-columns:1fr}}
+    :root{color-scheme:dark;--bg:#070711;--panel:#111120;--panel2:#17172a;--line:#292945;--text:#f8f7ff;--muted:#9892b8;--soft:#c7c2e6;--good:#22c55e;--warn:#f5c542;--bad:#fb7185;--blue:#60a5fa;--violet:#a78bfa}*{box-sizing:border-box}body{margin:0;min-height:100vh;background:radial-gradient(circle at top left,#25145a 0,#070711 34%,#070711 100%);font-family:Inter,ui-sans-serif,system-ui,-apple-system,sans-serif;color:var(--text);font-size:13px}main{width:min(1280px,calc(100% - 28px));margin:0 auto;padding:28px 0 44px}.top{display:flex;align-items:flex-start;justify-content:space-between;gap:18px;margin-bottom:18px}.eyebrow{font-size:.7rem;letter-spacing:.18em;text-transform:uppercase;color:var(--muted);font-weight:900}h1{margin:6px 0 8px;font-size:clamp(1.6rem,4vw,2.7rem);line-height:1}.sub{color:var(--soft);max-width:860px;line-height:1.55}.actions{display:flex;gap:10px;flex-wrap:wrap}.btn{border:1px solid var(--line);background:rgba(255,255,255,.06);color:var(--text);border-radius:11px;padding:10px 14px;text-decoration:none;font-weight:850;cursor:pointer}.btn.primary{background:#fff;color:#050510}.btn:hover{filter:brightness(1.08)}.grid{display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:12px;margin:18px 0}.card{background:linear-gradient(180deg,rgba(255,255,255,.055),rgba(255,255,255,.025));border:1px solid var(--line);border-radius:18px;padding:16px;box-shadow:0 18px 45px rgba(0,0,0,.22)}.metric-title{color:var(--muted);text-transform:uppercase;letter-spacing:.16em;font-size:.66rem;font-weight:900}.metric-value{font-size:1.65rem;font-weight:950;margin-top:8px}.metric-detail{color:var(--muted);margin-top:6px;line-height:1.45}.ok{color:var(--good)}.warn{color:var(--warn)}.bad{color:var(--bad)}.blue{color:var(--blue)}.layout{display:grid;grid-template-columns:340px minmax(0,1fr);gap:14px}.panel-title{display:flex;justify-content:space-between;align-items:center;gap:10px;margin-bottom:12px}.panel-title h2{font-size:1rem;margin:0}.pill{display:inline-flex;align-items:center;gap:6px;border:1px solid var(--line);border-radius:999px;padding:5px 9px;color:var(--soft);background:rgba(255,255,255,.035);font-size:.72rem;font-weight:850}.dot{width:7px;height:7px;border-radius:50%;background:var(--muted)}.dot.live{background:var(--good);box-shadow:0 0 15px var(--good)}.providers{display:flex;flex-direction:column;gap:9px}.provider{border:1px solid var(--line);border-radius:14px;background:rgba(0,0,0,.12);padding:12px;cursor:pointer}.provider.active{border-color:var(--blue);box-shadow:0 0 0 1px rgba(96,165,250,.25)}.provider-top{display:flex;justify-content:space-between;align-items:center;gap:12px}.provider-name{font-weight:950}.provider-meta{color:var(--muted);font-size:.78rem;margin-top:5px}.bar{height:8px;background:#22223a;border-radius:999px;overflow:hidden;margin-top:10px}.bar>span{display:block;height:100%;background:linear-gradient(90deg,var(--violet),var(--blue));border-radius:999px}.toolbar{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px}.input{flex:1;min-width:180px;border:1px solid var(--line);background:#0c0c18;color:var(--text);border-radius:11px;padding:10px 12px;outline:none}.select{border:1px solid var(--line);background:#0c0c18;color:var(--text);border-radius:11px;padding:10px 12px}.toggle{display:flex;align-items:center;gap:7px;color:var(--soft);font-weight:800}.events{display:flex;flex-direction:column;gap:9px;max-height:620px;overflow:auto;padding-right:4px}.event{border:1px solid var(--line);border-left-width:4px;border-radius:14px;background:rgba(0,0,0,.16);padding:12px;display:grid;grid-template-columns:155px minmax(0,1fr) auto;gap:12px}.event.pick,.event.sticky_pick,.event.pick_retry_fresh,.event.model_detected{border-left-color:var(--blue)}.event.success{border-left-color:var(--good)}.event.rate_limited,.event.auth_failed,.event.transport_aborted,.event.all_suspended_pick,.event.all_suspended_withheld,.event.client_error,.event.provider_error{border-left-color:var(--warn)}.event.network_retryable,.event.transient_status,.event.saturated_reuse,.event.sticky_saturated_reuse,.event.sticky_saturated_rotate{border-left-color:var(--bad)}.event.inflight_timeout,.event.inflight_lease_expired{border-left-color:var(--muted)}.time{color:var(--muted);font-variant-numeric:tabular-nums}.etype{font-weight:950}.msg{color:var(--soft);line-height:1.45;word-break:break-word}.key,.mono{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;color:#e9d5ff}.empty{padding:34px;text-align:center;color:var(--muted);border:1px dashed var(--line);border-radius:16px}.foot{color:var(--muted);margin-top:16px;line-height:1.55}.kbd{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;background:#23233a;border:1px solid #363653;border-radius:7px;padding:2px 6px;color:var(--text)}.detail-grid{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:10px;margin-bottom:12px}.mini{border:1px solid var(--line);border-radius:13px;padding:11px;background:rgba(0,0,0,.12)}.mini b{display:block;font-size:1.15rem;margin-top:4px}.key-table-wrap{overflow:auto}.key-table{width:100%;border-collapse:separate;border-spacing:0 8px;min-width:780px}.key-table th{color:var(--muted);font-size:.68rem;text-align:left;text-transform:uppercase;letter-spacing:.12em}.key-table td{background:rgba(0,0,0,.15);border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:10px;vertical-align:top}.key-table td:first-child{border-left:1px solid var(--line);border-radius:12px 0 0 12px}.key-table td:last-child{border-right:1px solid var(--line);border-radius:0 12px 12px 0}.status{font-weight:900}.status.used{color:var(--good)}.status.unused{color:var(--muted)}.model-chip{display:inline-flex;margin:2px 4px 2px 0;padding:4px 7px;border:1px solid var(--line);border-radius:999px;color:var(--soft);background:rgba(255,255,255,.035);font-size:.72rem}.section{margin-top:14px}.session-note{border:1px solid rgba(96,165,250,.28);background:rgba(96,165,250,.08);border-radius:14px;padding:12px;color:var(--soft);margin-bottom:12px}.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0}@media(max-width:1050px){.grid{grid-template-columns:repeat(2,minmax(0,1fr))}.detail-grid{grid-template-columns:repeat(2,minmax(0,1fr))}.layout{grid-template-columns:1fr}.top{flex-direction:column}.event{grid-template-columns:1fr}.events{max-height:none}}@media(max-width:620px){.grid,.detail-grid{grid-template-columns:1fr}}
   </style>
 </head>
 <body>
@@ -62,15 +62,15 @@ function eventClass(v){return String(v||'event').replace(/[^a-z0-9_-]/gi,'_');}
 function fmtTime(ts){const d=new Date(ts);return isNaN(d)?'—':d.toLocaleString();}
 function isKeyEvent(e){return e.provider&&e.slot&&(e.key||e.type==='all_suspended_withheld');}
 function keyId(provider,slot,key){return `${provider}::${slot||''}::${key||''}`;}
-function eventText(e){const bits=[];if(e.provider)bits.push(e.provider);if(e.slot)bits.push(`#${e.slot}/${e.total}`);if(e.key)bits.push(e.key);if(e.model)bits.push(`model=${e.model}`);if(e.status)bits.push(`status=${e.status}`);if(e.errorStatus)bits.push(`error=${e.errorStatus}`);if(e.errorReason)bits.push(`reason=${e.errorReason}`);if(e.errorType)bits.push(`type=${e.errorType}`);if(e.waitMs)bits.push(`wait=${Math.round(e.waitMs/1000)}s`);if(e.code)bits.push(`code=${e.code}`);if(e.errorCode)bits.push(`errorCode=${e.errorCode}`);if(e.inflight)bits.push(`inflight=${e.inflight}/${e.maxInflight||'?'}`);return bits.join(' · ');}
-function emptyStat(provider,slot,total,key){return{provider,slot,total,key,picks:0,success:0,rate:0,retry:0,auth:0,transient:0,abort:0,last:'',lastPick:'',models:new Map()};}
-function buildStats(){const stats=new Map();for(const p of state.providers){for(const k of (p.keys||[]))stats.set(keyId(p.name,k.slot,k.key),emptyStat(p.name,k.slot,k.total,k.key));}for(const e of state.events){if(!isKeyEvent(e))continue;const id=keyId(e.provider,e.slot,e.key);if(!stats.has(id))stats.set(id,emptyStat(e.provider,e.slot,e.total,e.key||'***'));const s=stats.get(id);const isPick=PICK_TYPES.includes(e.type);const isOutcome=['success','rate_limited','network_retryable','auth_failed','transient_status','inflight_timeout','transport_aborted'].includes(e.type);if(isPick){s.picks++;s.lastPick=e.ts||s.lastPick;}if(e.type==='success')s.success++;if(e.type==='rate_limited')s.rate++;if(e.type==='network_retryable')s.retry++;if(e.type==='auth_failed')s.auth++;if(e.type==='transport_aborted')s.abort++;if(e.type==='transient_status'||e.type==='inflight_timeout')s.transient++;s.last=e.ts||s.last;if(e.model){let m=s.models.get(e.model);if(!m)m={picks:0,success:0,rate:0,retry:0,auth:0,transient:0,abort:0,last:'',lastPick:'',observed:false};if(e.type==='model_detected')m.observed=true;if(isPick){m.picks++;m.lastPick=e.ts||m.lastPick;}if(e.type==='success')m.success++;if(e.type==='rate_limited')m.rate++;if(e.type==='network_retryable')m.retry++;if(e.type==='auth_failed')m.auth++;if(e.type==='transport_aborted')m.abort++;if(e.type==='transient_status'||e.type==='inflight_timeout')m.transient++;m.last=e.ts||m.last;s.models.set(e.model,m);}else if(isPick||isOutcome){s.unscoped=s.unscoped||{picks:0,success:0,rate:0,retry:0,auth:0,transient:0,abort:0,last:'',lastPick:''};if(isPick){s.unscoped.picks++;s.unscoped.lastPick=e.ts||s.unscoped.lastPick;}if(e.type==='success')s.unscoped.success++;if(e.type==='rate_limited')s.unscoped.rate++;if(e.type==='network_retryable')s.unscoped.retry++;if(e.type==='auth_failed')s.unscoped.auth++;if(e.type==='transport_aborted')s.unscoped.abort++;if(e.type==='transient_status'||e.type==='inflight_timeout')s.unscoped.transient++;s.unscoped.last=e.ts||s.unscoped.last;}}const now=Date.now();for(const s of stats.values()){ageStalePending(s,now);for(const m of s.models.values())ageStalePending(m,now);if(s.unscoped)ageStalePending(s.unscoped,now);}return stats;}
-function ageStalePending(v,now){const pending=pendingCount(v);if(!pending)return;const ts=Date.parse(v.lastPick||v.last||'');if(Number.isFinite(ts)&&now-ts>STALE_PENDING_MS){v.transient=(v.transient||0)+pending;v.stalePending=(v.stalePending||0)+pending;}}
+function eventText(e){const bits=[];if(e.provider)bits.push(e.provider);if(e.slot)bits.push(`#${e.slot}/${e.total}`);if(e.key)bits.push(e.key);if(e.model)bits.push(`model=${e.model}`);if(e.status)bits.push(`status=${e.status}`);if(e.errorStatus)bits.push(`error=${e.errorStatus}`);if(e.errorReason)bits.push(`reason=${e.errorReason}`);if(e.errorType)bits.push(`type=${e.errorType}`);if(e.source)bits.push(`source=${e.source}`);if(e.name)bits.push(`name=${e.name}`);if(e.waitMs)bits.push(`wait=${Math.round(e.waitMs/1000)}s`);if(e.code)bits.push(`code=${e.code}`);if(e.errorCode)bits.push(`errorCode=${e.errorCode}`);if(e.message)bits.push(`message=${e.message}`);if(e.inflight)bits.push(`inflight=${e.inflight}/${e.maxInflight||'?'}`);return bits.join(' · ');}
+function emptyStat(provider,slot,total,key){return{provider,slot,total,key,picks:0,success:0,rate:0,retry:0,auth:0,transient:0,abort:0,other:0,lease:0,last:'',lastPick:'',models:new Map()};}
+function buildStats(){const stats=new Map();for(const p of state.providers){for(const k of (p.keys||[]))stats.set(keyId(p.name,k.slot,k.key),emptyStat(p.name,k.slot,k.total,k.key));}for(const e of state.events){if(!isKeyEvent(e))continue;const id=keyId(e.provider,e.slot,e.key);if(!stats.has(id))stats.set(id,emptyStat(e.provider,e.slot,e.total,e.key||'***'));const s=stats.get(id);const isPick=PICK_TYPES.includes(e.type);const isOutcome=['success','rate_limited','network_retryable','auth_failed','transient_status','transport_aborted','client_error','provider_error'].includes(e.type),isLease=e.type==='inflight_timeout'||e.type==='inflight_lease_expired';if(isPick){s.picks++;s.lastPick=e.ts||s.lastPick;}if(e.type==='success')s.success++;if(e.type==='rate_limited')s.rate++;if(e.type==='network_retryable')s.retry++;if(e.type==='auth_failed')s.auth++;if(e.type==='transport_aborted')s.abort++;if(e.type==='client_error'||e.type==='provider_error')s.other++;if(isLease)s.lease++;if(e.type==='transient_status')s.transient++;s.last=e.ts||s.last;if(e.model){let m=s.models.get(e.model);if(!m)m={picks:0,success:0,rate:0,retry:0,auth:0,transient:0,abort:0,other:0,lease:0,last:'',lastPick:'',observed:false};if(e.type==='model_detected')m.observed=true;if(isPick){m.picks++;m.lastPick=e.ts||m.lastPick;}if(e.type==='success')m.success++;if(e.type==='rate_limited')m.rate++;if(e.type==='network_retryable')m.retry++;if(e.type==='auth_failed')m.auth++;if(e.type==='transport_aborted')m.abort++;if(e.type==='client_error'||e.type==='provider_error')m.other++;if(isLease)m.lease++;if(e.type==='transient_status')m.transient++;m.last=e.ts||m.last;s.models.set(e.model,m);}else if(isPick||isOutcome){s.unscoped=s.unscoped||{picks:0,success:0,rate:0,retry:0,auth:0,transient:0,abort:0,other:0,lease:0,last:'',lastPick:''};if(isPick){s.unscoped.picks++;s.unscoped.lastPick=e.ts||s.unscoped.lastPick;}if(e.type==='success')s.unscoped.success++;if(e.type==='rate_limited')s.unscoped.rate++;if(e.type==='network_retryable')s.unscoped.retry++;if(e.type==='auth_failed')s.unscoped.auth++;if(e.type==='transport_aborted')s.unscoped.abort++;if(e.type==='client_error'||e.type==='provider_error')s.unscoped.other++;if(isLease)s.unscoped.lease++;if(e.type==='transient_status')s.unscoped.transient++;s.unscoped.last=e.ts||s.unscoped.last;}}const now=Date.now();for(const s of stats.values()){ageStalePending(s,now);for(const m of s.models.values())ageStalePending(m,now);if(s.unscoped)ageStalePending(s.unscoped,now);}return stats;}
+function ageStalePending(v,now){const pending=pendingCount(v);if(!pending)return;const ts=Date.parse(v.lastPick||v.last||'');if(Number.isFinite(ts)&&now-ts>STALE_PENDING_MS){v.stalePending=(v.stalePending||0)+pending;}}
 function providerRows(provider,stats){return (provider?.keys||[]).map(k=>stats.get(keyId(provider.name,k.slot,k.key))||emptyStat(provider.name,k.slot,k.total,k.key));}
-function pendingCount(v){return Math.max(0,(v.picks||0)-((v.success||0)+(v.rate||0)+(v.retry||0)+(v.transient||0)+(v.auth||0)+(v.abort||0)));}
-function modelChips(row){const entries=[...row.models.entries()];const un=row.unscoped||{picks:0,success:0,rate:0,retry:0,auth:0,transient:0,abort:0};const unPending=pendingCount(un);if(!entries.length){const total=row.picks+row.success+row.rate+row.retry+row.transient+row.auth+(row.abort||0);return total?`<span class="model-chip">unscoped · p:${row.picks} ok:${row.success} pending:${pendingCount(row)} rl:${row.rate} retry:${row.retry+row.transient}${row.abort?` abort:${row.abort}`:''}</span>`:'<span class="model-chip">no model events yet</span>';}const chips=entries.map(([m,v])=>{const onlyObserved=v.observed&&!v.picks&&!v.success&&!v.rate&&!v.retry&&!v.transient&&!v.auth&&!v.abort;return `<span class="model-chip">${esc(m)} · ${onlyObserved?'observed':`p:${v.picks} ok:${v.success} pending:${pendingCount(v)} rl:${v.rate} retry:${v.retry+v.transient}${v.abort?` abort:${v.abort}`:''}`}</span>`;});if(un.picks||un.success||un.rate||un.retry||un.transient||un.auth||un.abort)chips.push(`<span class="model-chip">unscoped totals · p:${un.picks} ok:${un.success} pending:${unPending} rl:${un.rate} retry:${un.retry+un.transient}${un.abort?` abort:${un.abort}`:''}</span>`);return chips.join('');}
+function pendingCount(v){return Math.max(0,(v.picks||0)-((v.success||0)+(v.rate||0)+(v.retry||0)+(v.transient||0)+(v.auth||0)+(v.abort||0)+(v.other||0)));}
+function modelChips(row){const entries=[...row.models.entries()];const un=row.unscoped||{picks:0,success:0,rate:0,retry:0,auth:0,transient:0,abort:0,other:0,lease:0};const unPending=pendingCount(un);if(!entries.length){const total=row.picks+row.success+row.rate+row.retry+row.transient+row.auth+(row.abort||0)+(row.other||0);return total?`<span class="model-chip">unscoped · p:${row.picks} ok:${row.success} pending:${pendingCount(row)} rl:${row.rate} retry:${row.retry+row.transient}${row.abort?` abort:${row.abort}`:''}${row.other?` other:${row.other}`:''}${row.lease?` lease:${row.lease}`:''}</span>`:'<span class="model-chip">no model events yet</span>';}const chips=entries.map(([m,v])=>{const onlyObserved=v.observed&&!v.picks&&!v.success&&!v.rate&&!v.retry&&!v.transient&&!v.auth&&!v.abort&&!v.other;return `<span class="model-chip">${esc(m)} · ${onlyObserved?'observed':`p:${v.picks} ok:${v.success} pending:${pendingCount(v)} rl:${v.rate} retry:${v.retry+v.transient}${v.abort?` abort:${v.abort}`:''}${v.other?` other:${v.other}`:''}${v.lease?` lease:${v.lease}`:''}`}</span>`;});if(un.picks||un.success||un.rate||un.retry||un.transient||un.auth||un.abort||un.other)chips.push(`<span class="model-chip">unscoped totals · p:${un.picks} ok:${un.success} pending:${unPending} rl:${un.rate} retry:${un.retry+un.transient}${un.abort?` abort:${un.abort}`:''}${un.other?` other:${un.other}`:''}${un.lease?` lease:${un.lease}`:''}</span>`);return chips.join('');}
 function routeNote(provider){const routes=(state.runtime&&state.runtime.routes)||[],hits=routes.filter(r=>r.provider===provider.name);if(!routes.length)return '<div class="session-note">Current LLM route is unknown to this view. Usage appears only after OpenClaw sends a request to this provider hostname.</div>';if(hits.length)return `<div class="session-note"><b>Active route:</b> ${hits.map(r=>`${esc(r.role)} ${esc(r.model)}`).join(' · ')}. New messages using this route should create pick/success events below.</div>`;return `<div class="session-note bad"><b>Not on current model route.</b> Current route is ${routes.map(r=>`${esc(r.role)} ${esc(r.model)} → ${esc(r.provider)}`).join(' · ')}. ${esc(provider.name)} keys stay unused until you select a ${esc(provider.name)} model or a fallback reaches it.</div>`;}
-function renderProviderDetails(provider,stats){if(!provider){$('detailTitle').textContent='Provider details';$('details').innerHTML='<div class="empty">Select a provider to see key/session usage.</div>';return;}const rows=providerRows(provider,stats),used=rows.filter(r=>r.picks||r.success).length,rates=rows.reduce((a,r)=>a+r.rate,0),retries=rows.reduce((a,r)=>a+r.retry+r.transient,0),auths=rows.reduce((a,r)=>a+r.auth,0),aborts=rows.reduce((a,r)=>a+(r.abort||0),0),pending=rows.reduce((a,r)=>a+pendingCount(r),0);$('detailTitle').textContent=`${provider.name} · provider session`;const modelSet=new Set();rows.forEach(r=>r.models.forEach((_,m)=>modelSet.add(m)));const table=rows.map(r=>{const usedKey=(r.picks||r.success)>0,pending=pendingCount(r);return `<tr><td><b class="mono">#${r.slot}/${r.total}</b><div class="key">${esc(r.key)}</div></td><td><span class="status ${usedKey?'used':'unused'}">${usedKey?'USED':'UNUSED'}</span><div class="metric-detail">last: ${esc(r.last?fmtTime(r.last):'never')}</div>${pending?`<div class="metric-detail warn">pending/no response observed: ${pending}</div>`:''}</td><td>pick ${r.picks}<br>success ${r.success}<br>pending ${pending}<br>rate ${r.rate}</td><td>retry ${r.retry}<br>transient ${r.transient}<br>auth ${r.auth}<br>abort ${r.abort||0}</td><td>${modelChips(r)}</td></tr>`;}).join('');$('details').innerHTML=`${routeNote(provider)}<div class="session-note">Session: <b>${esc(provider.name)}</b>. Raw keys are never returned here; this page only shows masked keys and slot numbers from the configured pool.</div><div class="detail-grid"><div class="mini"><span class="metric-title">Total keys</span><b>${provider.total||0}</b></div><div class="mini"><span class="metric-title">Used / unused</span><b>${used}/${Math.max(0,(provider.total||0)-used)}</b></div><div class="mini"><span class="metric-title">Rate / pending</span><b>${rates}/${pending}</b></div><div class="mini"><span class="metric-title">Retry/auth/abort</span><b>${retries}/${auths}/${aborts}</b></div></div>${provider.name==='gemini'?`<p class="foot">Gemini quota is tracked per key + model. Models seen: ${modelSet.size?[...modelSet].map(m=>`<span class="model-chip">${esc(m)}</span>`).join(''):'<span class="kbd">none yet — new requests will show per-model once the rotator can read the request body</span>'}</p>`:''}<div class="key-table-wrap"><table class="key-table"><thead><tr><th>Hidden key slot</th><th>Status</th><th>Usage</th><th>Failures / aborts</th><th>Per-model / details</th></tr></thead><tbody>${table||'<tr><td colspan="5">No keys configured</td></tr>'}</tbody></table></div>`;}
+function renderProviderDetails(provider,stats){if(!provider){$('detailTitle').textContent='Provider details';$('details').innerHTML='<div class="empty">Select a provider to see key/session usage.</div>';return;}const rows=providerRows(provider,stats),used=rows.filter(r=>r.picks||r.success).length,rates=rows.reduce((a,r)=>a+r.rate,0),retries=rows.reduce((a,r)=>a+r.retry+r.transient,0),auths=rows.reduce((a,r)=>a+r.auth,0),aborts=rows.reduce((a,r)=>a+(r.abort||0),0),pending=rows.reduce((a,r)=>a+pendingCount(r),0);$('detailTitle').textContent=`${provider.name} · provider session`;const modelSet=new Set();rows.forEach(r=>r.models.forEach((_,m)=>modelSet.add(m)));const table=rows.map(r=>{const usedKey=(r.picks||r.success)>0,pending=pendingCount(r),stale=r.stalePending||0;return `<tr><td><b class="mono">#${r.slot}/${r.total}</b><div class="key">${esc(r.key)}</div></td><td><span class="status ${usedKey?'used':'unused'}">${usedKey?'USED':'UNUSED'}</span><div class="metric-detail">last: ${esc(r.last?fmtTime(r.last):'never')}</div>${pending?`<div class="metric-detail warn">pending/no response observed: ${pending}${stale?` (stale ${stale})`:''}</div>`:''}</td><td>pick ${r.picks}<br>success ${r.success}<br>pending ${pending}<br>rate ${r.rate}</td><td>retry ${r.retry}<br>transient ${r.transient}<br>auth ${r.auth}<br>abort ${r.abort||0}<br>other ${r.other||0}<br>lease ${r.lease||0}</td><td>${modelChips(r)}</td></tr>`;}).join('');$('details').innerHTML=`${routeNote(provider)}<div class="session-note">Session: <b>${esc(provider.name)}</b>. Raw keys are never returned here; this page only shows masked keys and slot numbers from the configured pool.</div><div class="detail-grid"><div class="mini"><span class="metric-title">Total keys</span><b>${provider.total||0}</b></div><div class="mini"><span class="metric-title">Used / unused</span><b>${used}/${Math.max(0,(provider.total||0)-used)}</b></div><div class="mini"><span class="metric-title">Rate / pending</span><b>${rates}/${pending}</b></div><div class="mini"><span class="metric-title">Retry/auth/abort</span><b>${retries}/${auths}/${aborts}</b></div></div>${provider.name==='gemini'?`<p class="foot">Gemini quota is tracked per key + model. Models seen: ${modelSet.size?[...modelSet].map(m=>`<span class="model-chip">${esc(m)}</span>`).join(''):'<span class="kbd">none yet — new requests will show per-model once the rotator can read the request body</span>'}</p>`:''}<div class="key-table-wrap"><table class="key-table"><thead><tr><th>Hidden key slot</th><th>Status</th><th>Usage</th><th>Failures / aborts</th><th>Per-model / details</th></tr></thead><tbody>${table||'<tr><td colspan="5">No keys configured</td></tr>'}</tbody></table></div>`;}
 function render(){const q=$('q').value.trim().toLowerCase(),type=$('type').value,providers=state.providers||[],events=state.events||[],stats=buildStats();if(!providers.some(p=>p.name===state.selected)&&providers[0])state.selected=providers[0].name;const totalKeys=providers.reduce((a,p)=>a+(Number(p.total)||0),0),usedKeyCount=[...stats.values()].filter(s=>s.picks||s.success).length;$('mKeys').textContent=totalKeys;$('mUsed').textContent=usedKeyCount;$('mUnused').textContent=Math.max(0,totalKeys-usedKeyCount);$('mRate').textContent=events.filter(e=>e.type==='rate_limited').length;$('mUpdate').textContent=new Date().toLocaleTimeString();$('providerCount').textContent=`${providers.length} active`;$('eventCount').textContent=`${events.length} events`;$('providers').innerHTML=providers.length?providers.map(p=>{const rows=providerRows(p,stats),used=rows.filter(r=>r.picks||r.success).length,total=Number(p.total)||0,aliases=p.aliases?' · '+esc(p.aliases):'';return `<button class="provider ${p.name===state.selected?'active':''}" data-provider="${esc(p.name)}" type="button"><div class="provider-top"><div class="provider-name">${esc(p.name)}</div><span class="pill">${used}/${total} used</span></div><div class="provider-meta">${esc(p.env||'configured pool')}${aliases}</div><div class="bar"><span style="width:${total?Math.round((used/total)*100):0}%"></span></div></button>`;}).join(''):'<div class="empty">No key pools detected in this process env.</div>';$('providers').querySelectorAll('[data-provider]').forEach(el=>el.onclick=()=>{state.selected=el.dataset.provider;localStorage.setItem('hc.keyRotator.provider',state.selected);render();});renderProviderDetails(providers.find(p=>p.name===state.selected)||providers[0],stats);let filtered=events.slice().reverse().filter(e=>(!type||e.type===type));if($('scope').checked&&state.selected)filtered=filtered.filter(e=>!e.provider||e.provider==='system'||e.provider===state.selected);if(q)filtered=filtered.filter(e=>JSON.stringify(e).toLowerCase().includes(q));$('events').innerHTML=filtered.length?filtered.map(e=>`<article class="event ${eventClass(e.type)}"><div class="time">${esc(fmtTime(e.ts))}</div><div><div class="etype">${esc(e.type||'event')}</div><div class="msg">${esc(eventText(e)||'system event')}</div></div><div class="key">${esc(e.key||'')}</div></article>`).join(''):'<div class="empty">No matching events yet. Send a message or disable provider scope to view other providers.</div>';}
 async function load(){if(state.paused)return;try{const r=await fetch('/api/key-rotator/logs?limit=1000',{cache:'no-store'});if(r.status===401){location.href='/login?next='+encodeURIComponent('/key-rotator');return;}const d=await r.json();state.events=d.events||[];state.providers=d.providers||[];state.runtime=d.runtime||{routes:[]};state.lastPayload=d;const log=d.log||{};$('source').textContent=log.exists?`${d.file||'event log'} · ${Math.round((log.size||0)/1024)} KiB`:`${d.file||'event log'} · waiting`;render();}catch(e){$('events').innerHTML='<div class="empty bad">Could not load rotator logs: '+esc(e.message)+'</div>';}}
 $('q').addEventListener('input',render);$('type').addEventListener('change',render);$('scope').addEventListener('change',render);$('refresh').onclick=load;$('pause').onclick=()=>{state.paused=!state.paused;$('pause').textContent=state.paused?'Resume':'Pause';$('liveDot').classList.toggle('live',!state.paused);$('liveText').textContent=state.paused?'Paused':'Live';};$('export').onclick=()=>{const data=JSON.stringify(state.lastPayload||{providers:state.providers,events:state.events},null,2),blob=new Blob([data],{type:'application/json'}),url=URL.createObjectURL(blob),a=document.createElement('a');a.href=url;a.download='huggingclaw-key-rotator.json';a.click();URL.revokeObjectURL(url);};load();setInterval(load,2500);

multi-provider-key-rotator.cjs

@@ -123,6 +123,10 @@ const ERROR_BODY_SNIFF_MAX_BYTES = Math.max(
   4 * 1024,
   Math.min(256 * 1024, parseInt(process.env.KEY_ERROR_BODY_SNIFF_MAX_BYTES || '', 10) || 64 * 1024),
 );
+const ERROR_BODY_WAIT_MS = Math.max(
+  250,
+  Math.min(10_000, parseInt(process.env.KEY_ERROR_BODY_WAIT_MS || '', 10) || 1500),
+);
 
 const USE_SUSPENDED_KEY_AS_LAST_RESORT = !/^(0|false|no|off)$/i.test(
   String(process.env.KEY_USE_SUSPENDED_AS_LAST_RESORT || 'true').trim(),
@@ -342,6 +346,22 @@ function normalizeErrorToken(value) {
   return String(value || '').trim().toLowerCase();
 }
 
+function retryDelayMsFromText(text) {
+  const raw = String(text || '');
+  if (!raw) return 0;
+  const retryIn = raw.match(/(?:retry|try again|retry-after|retry_after)[^0-9]{0,40}(\d+(?:\.\d+)?)\s*(ms|milliseconds?|s|sec|secs|seconds?|m|mins?|minutes?)?/i);
+  if (!retryIn) return 0;
+  const value = Number(retryIn[1]);
+  if (!Number.isFinite(value) || value < 0) return 0;
+  const unit = String(retryIn[2] || 's').toLowerCase();
+  const ms = unit.startsWith('m') && !unit.startsWith('ms') && !unit.startsWith('milli')
+    ? value * 60_000
+    : unit.startsWith('ms') || unit.startsWith('milli')
+      ? value
+      : value * 1000;
+  return Math.min(MAX_RETRY_AFTER_MS, Math.round(ms));
+}
+
 function parseProviderErrorInfo(text) {
   if (!text || typeof text !== 'string') return null;
   const info = { raw: text.slice(0, ERROR_BODY_SNIFF_MAX_BYTES) };
@@ -355,14 +375,19 @@ function parseProviderErrorInfo(text) {
       info.reason = err.reason ?? body.reason;
       info.message = err.message ?? body.message;
       if (!info.reason && Array.isArray(err.errors) && err.errors[0]) info.reason = err.errors[0].reason;
-      if (!info.reason && Array.isArray(err.details)) {
-        const quota = err.details.find(d => d && (d.reason || d.violations || d.quotaMetric));
-        if (quota) info.reason = quota.reason || quota.quotaMetric || 'quota_details';
+      if (Array.isArray(err.details)) {
+        if (!info.reason) {
+          const quota = err.details.find(d => d && (d.reason || d.violations || d.quotaMetric));
+          if (quota) info.reason = quota.reason || quota.quotaMetric || 'quota_details';
+        }
+        const retry = err.details.find(d => d && (d.retryDelay || d.retry_delay));
+        if (retry) info.retryAfterMs = retryDelayMsFromText(`retry in ${retry.retryDelay || retry.retry_delay}`);
       }
     }
   } catch (_) {
     info.message = text.slice(0, 512);
   }
+  if (!info.retryAfterMs) info.retryAfterMs = retryDelayMsFromText(info.message || text);
   return info;
 }
 
@@ -411,7 +436,54 @@ function providerErrorFields(errorInfo) {
   };
 }
 
+function extractStatusFromError(err) {
+  const direct = [
+    err?.status,
+    err?.statusCode,
+    err?.code,
+    err?.response?.status,
+    err?.response?.statusCode,
+    err?.cause?.status,
+    err?.cause?.statusCode,
+    err?.cause?.response?.status,
+    err?.cause?.response?.statusCode,
+  ];
+  for (const value of direct) {
+    if (typeof value === 'number' && value >= 100 && value <= 599) return value;
+    if (typeof value === 'string' && /^\d{3}$/.test(value.trim())) {
+      const n = Number(value.trim());
+      if (n >= 100 && n <= 599) return n;
+    }
+  }
+
+  const text = [
+    err?.message,
+    err?.cause?.message,
+    err?.stack,
+  ].filter(Boolean).join(' ');
+  const match = text.match(/(?:^|\D)([1-5]\d{2})\s*(?:status(?:\s+code)?|http|response|provider)/i)
+    || text.match(/(?:status(?:\s+code)?|http|response|provider)\D+([1-5]\d{2})(?:\D|$)/i)
+    || text.match(/^\s*(?:[A-Za-z][A-Za-z0-9_]*Error:\s*)?([1-5]\d{2})\b/)
+    || text.match(/\b([1-5]\d{2})\s+(?:RESOURCE_EXHAUSTED|UNAVAILABLE|INTERNAL|DEADLINE_EXCEEDED|PERMISSION_DENIED|UNAUTHENTICATED|INVALID_ARGUMENT|FAILED_PRECONDITION|NOT_FOUND|overloaded|too many|unauthori[sz]ed|forbidden|rate)/i);
+  return match ? Number(match[1]) : null;
+}
+
+function transportErrorInfo(err, status) {
+  const message = String(err?.message || err?.cause?.message || '').slice(0, 512);
+  const info = parseProviderErrorInfo(message) || { message };
+  if (!info.message) info.message = message;
+  if (typeof status === 'number') info.status = String(status);
+  if (err?.type) info.type = err.type;
+  if (err?.reason) info.reason = err.reason;
+  return info;
+}
+
 function statusNeedsErrorBodyForScope(status) {
+  // Wait for the body only where provider text can materially change key
+  // health classification (notably Google/Gemini 403 quota vs permission).
+  // 401/402/429/5xx are unambiguous from headers; handling them immediately
+  // avoids 30s lease-expiry noise when OpenClaw aborts/failover stops reading
+  // the error body.
   return status === 400 || status === 403 || status === 409 || status === 413 || status === 423 || status === 425 || status === 529;
 }
 
@@ -928,7 +1000,7 @@ function beginInFlight(p, key, model = null) {
       // as a key failure: one logical OpenClaw task can legitimately hold an
       // upstream stream longer than the lease TTL, and rotating/blacklisting on
       // a bookkeeping timeout can split that task across multiple API keys.
-      emitEvent('inflight_timeout', p, key, { model: token.model || null, inflightBefore: before, inflightAfter: next, ttlMs: INFLIGHT_TTL_MS, classifiedAs: 'lease_cleanup' });
+      emitEvent('inflight_lease_expired', p, key, { model: token.model || null, inflightBefore: before, inflightAfter: next, ttlMs: INFLIGHT_TTL_MS, classifiedAs: 'lease_cleanup' });
     }
   }, INFLIGHT_TTL_MS);
   token.timer.unref?.();
@@ -1147,10 +1219,11 @@ function setAuthHeader(headers, key) {
   return { authorization: val };
 }
 
-function handleStatus(p, key, status, model, retryAfterMs, errorInfo) {
+function handleStatus(p, key, status, model, retryAfterMs, errorInfo, extra = {}) {
   if (!p || !key) return;
+  retryAfterMs = retryAfterMs || errorInfo?.retryAfterMs || 0;
   const failureKind = classifyProviderFailure(p, status, errorInfo);
-  const errorFields = providerErrorFields(errorInfo);
+  const errorFields = { ...providerErrorFields(errorInfo), ...extra };
 
   if (failureKind === 'auth') {
     // Invalid/expired/unauthorized key — always a global (not model-scoped) blacklist.
@@ -1161,8 +1234,9 @@ function handleStatus(p, key, status, model, retryAfterMs, errorInfo) {
     ks.blacklistedUntil = Date.now() + PERM_SUSPEND_MS;
     clearStickyKey(p, key);
     clearTaskAffinityKey(p, key);
-    warn(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} auth-failed (${status}) — suspended for ${formatHours(PERM_SUSPEND_MS)} h`);
-    emitEvent('auth_failed', p, key, { status, suspendMs: PERM_SUSPEND_MS, ...errorFields });
+    const reason = errorFields.errorReason || errorFields.errorStatus || errorFields.errorType || errorFields.errorCode || errorFields.source;
+    warn(`[key-rotator] ${p.name}: ${keySlot(p, key)}${keyMask(key)} auth-failed (${status})${reason ? ` reason=${reason}` : ''}${model ? ` model=${model}` : ''} — suspended for ${formatHours(PERM_SUSPEND_MS)} h`);
+    emitEvent('auth_failed', p, key, { status, model, suspendMs: PERM_SUSPEND_MS, ...errorFields });
     return;
   }
 
@@ -1194,6 +1268,18 @@ function handleStatus(p, key, status, model, retryAfterMs, errorInfo) {
   if (status >= 200 && status < 400) {
     recordSuccess(p, key, model);
     emitEvent('success', p, key, { status, model });
+    return;
+  }
+
+  if (status >= 400) {
+    // Non-key failures (malformed request, missing model, unsupported feature,
+    // content/safety/request-size errors, etc.) still need an explicit outcome
+    // so the dashboard does not age a completed request into a fake transient.
+    // Do not blacklist or rotate on these: the selected API key may be healthy.
+    const eventType = status < 500 ? 'client_error' : 'provider_error';
+    const reason = errorFields.errorReason || errorFields.errorStatus || errorFields.errorType || errorFields.errorCode || errorFields.source;
+    warn(`[key-rotator] ${p.name}: ${eventType} status=${status}${reason ? ` reason=${reason}` : ''} on ${keySlot(p, key)}${keyMask(key)}${model ? ` model=${model}` : ''} (key not penalized)`);
+    emitEvent(eventType, p, key, { status, model, ...errorFields });
   }
 }
 
@@ -1202,13 +1288,36 @@ function handleTransportError(p, key, err, model = null) {
   // Node.js 18+ undici fetch throws TypeError: "fetch failed" where the actual
   // network error code lives in err.cause.code (e.g. ECONNRESET, ETIMEDOUT,
   // ENOTFOUND).  Fall back to err.cause.code so retryable network errors are
-  // correctly classified and transient blacklists are applied.
+  // correctly classified and transient blacklists are applied.  OpenClaw
+  // failover can also throw FailoverError("401 status code (no body)") instead
+  // of returning a Response; parse that embedded HTTP status so the selected
+  // key is marked auth/rate/transient correctly instead of becoming a stale
+  // pending pick on the dashboard.
   const code = (err?.code || err?.cause?.code)
     ? String(err.code || err.cause?.code).toUpperCase()
     : '';
   const name = String(err?.name || '');
   const message = String(err?.message || err?.cause?.message || '');
   const haystack = `${name} ${message}`.toLowerCase();
+  if (isCallerAbortError(err)) {
+    debug(`[key-rotator] ${p.name}: caller abort ${name || 'AbortError'} on ${keySlot(p, key)}${keyMask(key)}${model ? ` model=${model}` : ''} — leaving key sticky/healthy`);
+    emitEvent('transport_aborted', p, key, { model, name: name || 'AbortError', code, message: message.slice(0, 240), classifiedAs: 'caller_abort' });
+    return;
+  }
+
+  const embeddedStatus = extractStatusFromError(err);
+  if (embeddedStatus) {
+    const errorInfo = transportErrorInfo(err, embeddedStatus);
+    warn(`[key-rotator] ${p.name}: transport status=${embeddedStatus} ${name || 'Error'} on ${keySlot(p, key)}${keyMask(key)}${model ? ` model=${model}` : ''}${message ? ` message=${JSON.stringify(message.slice(0, 160))}` : ''}`);
+    handleStatus(p, key, embeddedStatus, model, 0, errorInfo, {
+      source: 'transport_error',
+      ...(name ? { name } : {}),
+      ...(code ? { code } : {}),
+      ...(message ? { message: message.slice(0, 240) } : {}),
+    });
+    return;
+  }
+
   const looksRateOrQuota = /rate.?limit|too.?many|quota|resource.?exhaust|usage.?limit|insufficient.?quota|capacity.?exceeded|tokens?.?per|requests?.?per|rate_limit|rate.?limited|userratelimit|dailylimit|limitexceeded/.test(haystack);
   if (looksRateOrQuota) {
     recordFailure(p, key, model, 0);
@@ -1218,11 +1327,6 @@ function handleTransportError(p, key, err, model = null) {
     emitEvent('rate_limited', p, key, { model, name: name || 'Error', code, source: 'transport_error', message: message.slice(0, 240) });
     return;
   }
-  if (isCallerAbortError(err)) {
-    debug(`[key-rotator] ${p.name}: caller abort ${name || 'AbortError'} on ${keySlot(p, key)}${keyMask(key)}${model ? ` model=${model}` : ''} — leaving key sticky/healthy`);
-    emitEvent('transport_aborted', p, key, { model, name: name || 'AbortError', code, message: message.slice(0, 240), classifiedAs: 'caller_abort' });
-    return;
-  }
   const retryable = shouldRetryTransportError(err, code);
   if (retryable) {
     recordTransientFailure(p, key, model);
@@ -1691,6 +1795,7 @@ function wrapUndiciHandler(handler, provider, key, inFlightToken, getModel) {
   let settled = false;
   let statusHandled = false;
   let errorBytes = 0;
+  let bodyWaitTimer = null;
   const errorChunks = [];
   const currentModel = () => (typeof getModel === 'function' ? getModel() : null);
   const collectErrorBody = (chunk) => {
@@ -1708,6 +1813,12 @@ function wrapUndiciHandler(handler, provider, key, inFlightToken, getModel) {
     if (!errorChunks.length) return null;
     try { return parseProviderErrorInfo(Buffer.concat(errorChunks).toString('utf8')); } catch (_) { return null; }
   };
+  const clearBodyWaitTimer = () => {
+    if (bodyWaitTimer) {
+      clearTimeout(bodyWaitTimer);
+      bodyWaitTimer = null;
+    }
+  };
   const handleHeadersStatus = (force = false) => {
     if (statusHandled || !statusCode) return false;
     // Success and unambiguous failures are emitted as soon as headers arrive.
@@ -1719,12 +1830,14 @@ function wrapUndiciHandler(handler, provider, key, inFlightToken, getModel) {
     // quota vs global auth suspension.
     if (!force && statusNeedsErrorBodyForScope(statusCode)) return false;
     statusHandled = true;
+    clearBodyWaitTimer();
     try { handleStatus(provider, key, statusCode, currentModel(), retryAfterMs, currentErrorInfo()); } catch (_) {}
     return true;
   };
   const settle = (fn) => {
     if (settled) return;
     settled = true;
+    clearBodyWaitTimer();
     try { endInFlight(provider, key, inFlightToken); } catch (_) {}
     try { fn(); } catch (_) {}
   };
@@ -1736,8 +1849,17 @@ function wrapUndiciHandler(handler, provider, key, inFlightToken, getModel) {
           retryAfterMs = parseRetryAfterMs(uGetHeader(headers, 'retry-after'));
           if (handleHeadersStatus()) {
             // Header-level status accounting is complete; close the in-flight
-            // token now so successful streams do not later report timeouts.
+            // token now so successful streams do not later report lease expiry.
             settle(() => {});
+          } else if (statusNeedsErrorBodyForScope(statusCode) && !bodyWaitTimer) {
+            // Some OpenClaw failover paths stop reading an error response once a
+            // higher layer has enough information to throw.  Do not let an
+            // ambiguous 4xx wait for the 30s in-flight lease: give the provider
+            // body a short chance to arrive, then classify with whatever we saw.
+            bodyWaitTimer = setTimeout(() => {
+              settle(() => { handleHeadersStatus(true); });
+            }, ERROR_BODY_WAIT_MS);
+            bodyWaitTimer.unref?.();
           }
           return target.onHeaders ? target.onHeaders.call(target, sc, headers, resume, statusMessage) : undefined;
         };
@@ -2336,7 +2458,7 @@ if (hasProviderKeys) {
   patchUndici();         // covers OpenClaw gateway's bundled undici AI calls
   startDiagnostics();
 
-  debug(`[key-rotator] loaded — cooldown base:${BASE_COOLDOWN_MS/1000}s max-strikes:${MAX_STRIKES} perm-suspend:${formatHours(PERM_SUSPEND_MS)}h (cap 16h) max-inflight-per-key:${MAX_INFLIGHT_PER_KEY} max-retry-after:${MAX_RETRY_AFTER_MS/1000}s max-key-wait:${MAX_KEY_WAIT_MS/1000}s diagnostics:${DIAGNOSTICS_ENABLED ? 'on' : 'off'} log-level:${LOG_LEVEL} verbose-picks:${VERBOSE_PICKS ? 'on' : 'off'} suspended-last-resort:${USE_SUSPENDED_KEY_AS_LAST_RESORT ? 'on' : 'off'} per-model-providers:${providerState.filter(p => p.perModelLimits).map(p => p.name).join(',') || 'none'} model-from-body:on model-sniff-max:${REQUEST_MODEL_SNIFF_MAX_BYTES} error-sniff-max:${ERROR_BODY_SNIFF_MAX_BYTES} inflight-ttl:${INFLIGHT_TTL_MS}ms task-affinity:${TASK_AFFINITY_MS}ms/${TASK_AFFINITY_MAX_REUSES}reuses sticky-until-failure:${STICKY_UNTIL_FAILURE ? 'on' : 'off'} sticky-scope:${String(process.env.KEY_STICKY_SCOPE || 'auto').trim().toLowerCase() || 'auto'} sticky-providers:${[...STICKY_PROVIDER_SET].join(',') || 'none'} llm-fallback-providers:${LLM_FALLBACK_PROVIDER_SET ? [...LLM_FALLBACK_PROVIDER_SET].join(',') : 'all'}`);
+  debug(`[key-rotator] loaded — cooldown base:${BASE_COOLDOWN_MS/1000}s max-strikes:${MAX_STRIKES} perm-suspend:${formatHours(PERM_SUSPEND_MS)}h (cap 16h) max-inflight-per-key:${MAX_INFLIGHT_PER_KEY} max-retry-after:${MAX_RETRY_AFTER_MS/1000}s max-key-wait:${MAX_KEY_WAIT_MS/1000}s diagnostics:${DIAGNOSTICS_ENABLED ? 'on' : 'off'} log-level:${LOG_LEVEL} verbose-picks:${VERBOSE_PICKS ? 'on' : 'off'} suspended-last-resort:${USE_SUSPENDED_KEY_AS_LAST_RESORT ? 'on' : 'off'} per-model-providers:${providerState.filter(p => p.perModelLimits).map(p => p.name).join(',') || 'none'} model-from-body:on model-sniff-max:${REQUEST_MODEL_SNIFF_MAX_BYTES} error-sniff-max:${ERROR_BODY_SNIFF_MAX_BYTES} error-body-wait:${ERROR_BODY_WAIT_MS}ms inflight-ttl:${INFLIGHT_TTL_MS}ms task-affinity:${TASK_AFFINITY_MS}ms/${TASK_AFFINITY_MAX_REUSES}reuses sticky-until-failure:${STICKY_UNTIL_FAILURE ? 'on' : 'off'} sticky-scope:${String(process.env.KEY_STICKY_SCOPE || 'auto').trim().toLowerCase() || 'auto'} sticky-providers:${[...STICKY_PROVIDER_SET].join(',') || 'none'} llm-fallback-providers:${LLM_FALLBACK_PROVIDER_SET ? [...LLM_FALLBACK_PROVIDER_SET].join(',') : 'all'}`);
   emitEvent('rotator_loaded', null, null, {
     providers: providerState.filter(p => p.keys.length).map(p => ({ name: p.name, total: p.keys.length })),
     logLevel: LOG_LEVEL,