fix: merge injected model providers using wildcard in start.sh

.env.example

@@ -169,7 +169,7 @@ LLM_API_KEY_FALLBACK_ENABLED=true
 # KEY_MAX_INFLIGHT_PER_KEY=3
 #
 # Auto-retry count for fetch requests on retryable errors/statuses.
-# Total attempts = 1 + retries (GET/HEAD/OPTIONS only to avoid duplicate writes).
+# Total attempts = 1 + retries (GET/HEAD/OPTIONS/POST).
 # KEY_FETCH_MAX_RETRIES=2
 #
 # Base delay (ms) between auto-retries. Exponential per attempt; also respects
@@ -180,6 +180,10 @@ LLM_API_KEY_FALLBACK_ENABLED=true
 # KEY_ROTATOR_DIAGNOSTICS=true
 # KEY_ROTATOR_DIAGNOSTICS_INTERVAL_MS=60000
 #
+# Log verbosity controls for rotator internals.
+# KEY_ROTATOR_LOG_LEVEL=info
+# KEY_ROTATOR_VERBOSE_PICKS=false
+#
 # Note: This rotator does not re-send the same failed request automatically.
 # It blacklists/penalizes the failed key so the *next* request prefers
 # healthier keys.

README.md

@@ -280,10 +280,12 @@ Optional tuning:
 - `KEY_PERM_SUSPEND_MS` (default `57600000`) — long suspend duration for exhausted/auth-invalid keys (**capped at 16h max**).
 - `KEY_FAILURE_DECAY_MS` (default `900000`) — recent-failure decay window used to deprioritize keys.
 - `KEY_MAX_INFLIGHT_PER_KEY` (default `3`) — soft concurrent request cap per key.
-- `KEY_FETCH_MAX_RETRIES` (default `2`) — auto-retry count for retryable failures on **GET/HEAD/OPTIONS** with a different key.
+- `KEY_FETCH_MAX_RETRIES` (default `2`) — auto-retry count for retryable failures on **GET/HEAD/OPTIONS/POST** with a different key.
 - `KEY_FETCH_RETRY_BASE_DELAY_MS` (default `250`) — base delay for retry backoff (respects `Retry-After`, capped to 10s).
 - `KEY_ROTATOR_DIAGNOSTICS=true` — emit periodic provider/key health snapshots.
 - `KEY_ROTATOR_DIAGNOSTICS_INTERVAL_MS` (default `60000`) — diagnostics interval.
+- `KEY_ROTATOR_LOG_LEVEL` (`info`/`debug`/`silent`, default `info`) — controls rotator log verbosity.
+- `KEY_ROTATOR_VERBOSE_PICKS` (`true`/`false`, default `false`) — enable per-request key-pick logs (best with `KEY_ROTATOR_LOG_LEVEL=debug`).
 
 Supported per-provider variables: `ANTHROPIC_API_KEYS`, `OPENAI_API_KEYS`, `GEMINI_API_KEYS`, `DEEPSEEK_API_KEYS`, `GROQ_API_KEYS`, `MISTRAL_API_KEYS`, `OPENROUTER_API_KEYS`, `XAI_API_KEYS`, `NVIDIA_API_KEYS`, `COHERE_API_KEYS`, `TOGETHER_API_KEYS`, `CEREBRAS_API_KEYS`, and more — see `.env.example` for the full list.
 

env-builder.js

@@ -529,7 +529,7 @@ const FIELDS = [
     "g": "Plugins",
     "icon": "🔄",
     "k": "KEY_FETCH_MAX_RETRIES",
-    "lbl": "Auto-retries for retryable failures (GET/HEAD/OPTIONS only)",
+    "lbl": "Auto-retries for retryable failures (GET/HEAD/OPTIONS/POST)",
     "type": "text",
     "ph": "2",
     "tag": "advanced"
@@ -543,6 +543,25 @@ const FIELDS = [
     "ph": "250",
     "tag": "advanced"
   },
+
+{
+    "g": "Plugins",
+    "icon": "🧾",
+    "k": "KEY_ROTATOR_LOG_LEVEL",
+    "lbl": "Key-rotator log level (info/debug/silent)",
+    "type": "text",
+    "ph": "info",
+    "tag": "advanced"
+  },
+{
+    "g": "Plugins",
+    "icon": "🧾",
+    "k": "KEY_ROTATOR_VERBOSE_PICKS",
+    "lbl": "Verbose per-request key pick logs (use with debug)",
+    "type": "toggle",
+    "ph": "false",
+    "tag": "advanced"
+  },
 {
     "g": "Plugins",
     "icon": "📊",

multi-provider-key-rotator.cjs

@@ -13,13 +13,18 @@
  *   KEY_BLACKLIST_COOLDOWN_MS   base backoff ms        (default 60 000)
  *   KEY_MAX_STRIKES             failures before perm   (default 3)
  *   LLM_API_KEY_FALLBACK_ENABLED true/false            (default true)
+ *   KEY_ROTATOR_LOG_LEVEL      info/debug/silent       (default info)
+ *   KEY_ROTATOR_VERBOSE_PICKS  true/false              (default false)
  */
 
 const http  = require('node:http');
 const https = require('node:https');
 
-const log  = (...a) => console.error(...a);
-const warn = (...a) => console.warn(...a);
+const LOG_LEVEL = String(process.env.KEY_ROTATOR_LOG_LEVEL || 'info').trim().toLowerCase();
+const VERBOSE_PICKS = /^(1|true|yes|on)$/i.test(String(process.env.KEY_ROTATOR_VERBOSE_PICKS || '').trim());
+const log  = (...a) => { if (LOG_LEVEL !== 'silent') console.error(...a); };
+const warn = (...a) => { if (LOG_LEVEL !== 'silent') console.warn(...a); };
+const debug = (...a) => { if (LOG_LEVEL === 'debug') console.error(...a); };
 
 // ─── Config ──────────────────────────────────────────────────────────────────
 
@@ -175,7 +180,7 @@ function isActive(p, key) {
   if (ks.blacklistedUntil === 0) return true;    // not blacklisted
   if (Date.now() >= ks.blacklistedUntil) {
     ks.blacklistedUntil = 0;                     // expired → back in pool
-    log(`[key-rotator] ${p.name}: ...${key.slice(-6)} back in pool`);
+    debug(`[key-rotator] ${p.name}: ...${key.slice(-6)} back in pool`);
     return true;
   }
   return false;
@@ -209,12 +214,27 @@ function recordFailure(p, key) {
     const jitter = 1 + ((Math.random() * 2 - 1) * (COOLDOWN_JITTER_PCT / 100));
     cooldown = Math.max(1000, Math.round(cooldown * jitter));
     const secs = Math.round(cooldown / 1000);
-    log(`[key-rotator] ${p.name}: ...${key.slice(-6)} strike ${ks.strikes}/${MAX_STRIKES} — backoff ${secs}s`);
+    debug(`[key-rotator] ${p.name}: ...${key.slice(-6)} strike ${ks.strikes}/${MAX_STRIKES} — backoff ${secs}s`);
   }
 
   ks.blacklistedUntil = Date.now() + cooldown;
 }
 
+/**
+ * Called on transient retryable failures (non-quota/rate):
+ * applies short cooldown without incrementing strikes.
+ */
+function recordTransientFailure(p, key) {
+  let ks = p.keyState.get(key);
+  if (!ks) { ks = makeKeyState(); p.keyState.set(key, ks); }
+  ks.lastFailureAt = Date.now();
+  const jitter = 1 + ((Math.random() * 2 - 1) * (COOLDOWN_JITTER_PCT / 100));
+  const cooldown = Math.max(1000, Math.round(BASE_COOLDOWN_MS * jitter));
+  ks.blacklistedUntil = Math.max(ks.blacklistedUntil || 0, Date.now() + cooldown);
+  const secs = Math.round(cooldown / 1000);
+  debug(`[key-rotator] ${p.name}: ...${key.slice(-6)} transient backoff ${secs}s (strikes unchanged)`);
+}
+
 /**
  * Called on any 2xx/3xx response — resets the key's strike counter.
  */
@@ -223,12 +243,12 @@ function recordSuccess(p, key) {
   if (ks && ks.strikes > 0) {
     ks.strikes = 0;
     ks.lastFailureAt = 0;
-    log(`[key-rotator] ${p.name}: ...${key.slice(-6)} recovered — strikes reset`);
+    debug(`[key-rotator] ${p.name}: ...${key.slice(-6)} recovered — strikes reset`);
   }
 }
 
 function classifyRetryableFailure(status, errCode) {
-  const retryableStatus = new Set([408, 425, 429, 500, 502, 503, 504, 529, 402]);
+  const retryableStatus = new Set([402, 408, 425, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524, 529]);
   const retryableErrorCodes = new Set([
     'ECONNRESET', 'ETIMEDOUT', 'EAI_AGAIN', 'ENOTFOUND',
     'ECONNREFUSED', 'EPIPE',
@@ -280,7 +300,7 @@ function nextKey(p) {
       const inflight = p.inFlight.get(key) || 0;
       if (inflight < MAX_INFLIGHT_PER_KEY) {
         p.idx = (i + 1) % total;   // next call starts AFTER the key we just picked
-        log(`[key-rotator] ${p.name}: picked ...${key.slice(-6)} inflight=${inflight + 1}/${MAX_INFLIGHT_PER_KEY}`);
+        if (VERBOSE_PICKS) debug(`[key-rotator] ${p.name}: picked ...${key.slice(-6)} inflight=${inflight + 1}/${MAX_INFLIGHT_PER_KEY}`);
         return key;
       }
       if (!bestPick) bestPick = { i, key, inflight, score: Number.POSITIVE_INFINITY };
@@ -353,10 +373,20 @@ function handleStatus(p, key, status) {
     warn(`[key-rotator] ${p.name}: ...${key.slice(-6)} auth-failed (${status}) — suspended for ${formatHours(PERM_SUSPEND_MS)} h`);
     return;
   }
-  if (classifyRetryableFailure(status)) {
+
+  if (status === 429 || status === 402) {
     recordFailure(p, key);
-    warn(`[key-rotator] ${p.name}: retryable status=${status} on ...${key.slice(-6)}`);
-  } else if (status >= 200 && status < 400) {
+    warn(`[key-rotator] ${p.name}: quota/rate status=${status} on ...${key.slice(-6)}`);
+    return;
+  }
+
+  if (classifyRetryableFailure(status)) {
+    recordTransientFailure(p, key);
+    warn(`[key-rotator] ${p.name}: transient status=${status} on ...${key.slice(-6)}`);
+    return;
+  }
+
+  if (status >= 200 && status < 400) {
     recordSuccess(p, key);
   }
 }
@@ -364,9 +394,11 @@ function handleStatus(p, key, status) {
 function handleTransportError(p, key, err) {
   if (!p || !key) return;
   const code = err?.code ? String(err.code).toUpperCase() : '';
-  if (classifyRetryableFailure(undefined, code)) {
+  const name = String(err?.name || '');
+  const retryable = classifyRetryableFailure(undefined, code) || name === 'AbortError';
+  if (retryable) {
     recordFailure(p, key);
-    warn(`[key-rotator] ${p.name}: retryable network code=${code} on ...${key.slice(-6)}`);
+    warn(`[key-rotator] ${p.name}: retryable network ${name || 'Error'}${code ? ` code=${code}` : ''} on ...${key.slice(-6)}`);
   }
 }
 
@@ -428,7 +460,8 @@ function patchFetch() {
       const baseRequest = new Request(input, init);
       const method = String(baseRequest.method || 'GET').toUpperCase();
       const replaySafe = method === 'GET' || method === 'HEAD' || method === 'OPTIONS';
-      const maxAttempts = replaySafe ? 1 + FETCH_MAX_RETRIES : 1;
+      const retryEligible = replaySafe || method === 'POST';
+      const maxAttempts = retryEligible ? 1 + FETCH_MAX_RETRIES : 1;
       const triedKeys = new Set();
       let lastErr = null;
       let lastResponse = null;
@@ -478,7 +511,7 @@ function patchFetch() {
               10_000,
               Math.max(retryAfterMs, FETCH_RETRY_BASE_DELAY_MS * Math.pow(2, attempt - 1)),
             );
-            warn(`[key-rotator] ${provider.name}: fetch retry ${attempt}/${maxAttempts - 1} after status=${response.status}`);
+            warn(`[key-rotator] ${provider.name}: fetch retry ${attempt}/${maxAttempts - 1} after status=${response.status} method=${method}`);
             await sleep(backoffMs);
             continue;
           }
@@ -488,10 +521,11 @@ function patchFetch() {
           try { handleTransportError(provider, usedKey, err); } catch (_) {}
           try { endInFlight(provider, usedKey); } catch (_) {}
           const code = err?.code ? String(err.code).toUpperCase() : '';
-          const shouldRetry = attempt < maxAttempts && classifyRetryableFailure(undefined, code);
+          const isAbort = String(err?.name || '') === 'AbortError';
+          const shouldRetry = attempt < maxAttempts && (classifyRetryableFailure(undefined, code) || isAbort);
           if (shouldRetry) {
             const backoffMs = Math.min(10_000, FETCH_RETRY_BASE_DELAY_MS * Math.pow(2, attempt - 1));
-            warn(`[key-rotator] ${provider.name}: fetch retry ${attempt}/${maxAttempts - 1} after network code=${code || 'unknown'}`);
+            warn(`[key-rotator] ${provider.name}: fetch retry ${attempt}/${maxAttempts - 1} after network ${isAbort ? 'AbortError' : `code=${code || 'unknown'}`} method=${method}`);
             await sleep(backoffMs);
             continue;
           }
@@ -578,4 +612,4 @@ patchHttpModule(http);
 patchHttpModule(https);
 startDiagnostics();
 
-log(`[key-rotator] loaded — cooldown base:${BASE_COOLDOWN_MS/1000}s max-strikes:${MAX_STRIKES} perm-suspend:${formatHours(PERM_SUSPEND_MS)}h (cap 16h) max-inflight-per-key:${MAX_INFLIGHT_PER_KEY} diagnostics:${DIAGNOSTICS_ENABLED ? 'on' : 'off'}`);
+log(`[key-rotator] loaded — cooldown base:${BASE_COOLDOWN_MS/1000}s max-strikes:${MAX_STRIKES} perm-suspend:${formatHours(PERM_SUSPEND_MS)}h (cap 16h) max-inflight-per-key:${MAX_INFLIGHT_PER_KEY} diagnostics:${DIAGNOSTICS_ENABLED ? 'on' : 'off'} log-level:${LOG_LEVEL} verbose-picks:${VERBOSE_PICKS ? 'on' : 'off'}`);

openclaw-sync.py

@@ -140,7 +140,8 @@ def copy_state_entry_with_retry(source_path: Path, backup_path: Path, attempts:
                 continue
             raise last_exc
 
-def snapshot_state_into_workspace() -> None:
+def snapshot_state_into_workspace() -> bool:
+    had_copy_failures = False
     try:
         STATE_DIR.mkdir(parents=True, exist_ok=True)
         # Atomic snapshot: copy to a staging dir first, then rename.
@@ -185,6 +186,7 @@ def snapshot_state_into_workspace() -> None:
         # known-good version for only those entries (staging was seeded from
         # previous backup). This preserves forward progress for the rest.
         if skipped_entries:
+            had_copy_failures = True
             for name, entry_exc in skipped_entries:
                 print(f"Warning: keeping previous state entry {name}: {entry_exc}")
             print(
@@ -200,10 +202,11 @@ def snapshot_state_into_workspace() -> None:
         if staging_dir.exists():
             shutil.rmtree(staging_dir, ignore_errors=True)
         print(f"Warning: could not snapshot OpenClaw state: {exc}")
+        had_copy_failures = True
 
     try:
         if not WHATSAPP_ENABLED:
-            return
+            return had_copy_failures
 
         STATE_DIR.mkdir(parents=True, exist_ok=True)
 
@@ -212,16 +215,16 @@ def snapshot_state_into_workspace() -> None:
                 shutil.rmtree(WHATSAPP_BACKUP_DIR, ignore_errors=True)
                 print("Removed backed-up WhatsApp credentials after reset request.")
             RESET_MARKER.unlink(missing_ok=True)
-            return
+            return had_copy_failures
 
         if not WHATSAPP_CREDS_DIR.exists():
-            return
+            return had_copy_failures
 
         file_count = count_files(WHATSAPP_CREDS_DIR)
         if file_count < 2:
             if file_count > 0:
                 print(f"WhatsApp backup skipped: credentials incomplete ({file_count} files).")
-            return
+            return had_copy_failures
 
         WHATSAPP_BACKUP_DIR.parent.mkdir(parents=True, exist_ok=True)
         if WHATSAPP_BACKUP_DIR.exists():
@@ -229,6 +232,8 @@ def snapshot_state_into_workspace() -> None:
         shutil.copytree(WHATSAPP_CREDS_DIR, WHATSAPP_BACKUP_DIR)
     except Exception as exc:
         print(f"Warning: could not snapshot WhatsApp state: {exc}")
+        had_copy_failures = True
+    return had_copy_failures
 
 
 def restore_embedded_state() -> None:
@@ -515,7 +520,7 @@ def _sync_once_unlocked(
         write_status("disabled", "HF_TOKEN is not configured.")
         return (last_fingerprint or "", last_marker or (0, 0, 0, ""))
 
-    snapshot_state_into_workspace()
+    had_snapshot_copy_failures = snapshot_state_into_workspace()
     repo_id = ensure_repo_exists()
     current_marker = metadata_marker(WORKSPACE)
     if last_marker is not None and current_marker == last_marker:
@@ -547,10 +552,13 @@ def _sync_once_unlocked(
                 commit_message=f"HuggingClaw sync {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}",
                 ignore_patterns=[".git/*", ".git"],
             )
-        try:
-            prune_remote_deleted_files(repo_id, snapshot_dir)
-        except Exception as prune_exc:
-            print(f"Warning: could not prune stale remote files: {prune_exc}")
+        if had_snapshot_copy_failures:
+            print("Warning: skipping remote prune this pass because local state snapshot had copy failures.")
+        else:
+            try:
+                prune_remote_deleted_files(repo_id, snapshot_dir)
+            except Exception as prune_exc:
+                print(f"Warning: could not prune stale remote files: {prune_exc}")
     finally:
         shutil.rmtree(snapshot_dir, ignore_errors=True)
 

start.sh

@@ -783,6 +783,36 @@ if [ "$WHATSAPP_ENABLED_NORMALIZED" = "true" ]; then
   CONFIG_JSON=$(echo "$CONFIG_JSON" | jq '.channels.whatsapp = {"dmPolicy": "pairing"}')
 fi
 
+
+validate_json_file() {
+  local file="$1"
+  [ -f "$file" ] || return 1
+  jq -e . "$file" >/dev/null 2>&1
+}
+
+write_json_atomic() {
+  local dest="$1"
+  local payload="$2"
+  local tmp
+  tmp="${dest}.tmp.$$"
+  printf '%s\n' "$payload" > "$tmp" || return 1
+  if ! jq -e . "$tmp" >/dev/null 2>&1; then
+    echo "ERROR: refusing to write invalid JSON to $dest" >&2
+    rm -f "$tmp"
+    return 1
+  fi
+  mv "$tmp" "$dest"
+}
+
+backup_config_copy() {
+  local src="$1"
+  [ -f "$src" ] || return 0
+  local stamp backup
+  stamp="$(date +%Y%m%d-%H%M%S)"
+  backup="${src}.backup.${stamp}"
+  cp -a "$src" "$backup" 2>/dev/null || cp "$src" "$backup" 2>/dev/null || true
+}
+
 # Write config
 EXISTING_CONFIG="/home/node/.openclaw/openclaw.json"
 WHATSAPP_CONFIG_ENABLED=false
@@ -820,8 +850,8 @@ if [ -f "$EXISTING_CONFIG" ]; then
      | if (($injectedModelsProviders | length) > 0) then
          ($injectedModelsProviders | to_entries) as $entries
          | reduce $entries[] as $e (.;
-             .models.providers[$e.key] = ((.models.providers[$e.key] // {})
-               + {models: (($e.value.models // []) | unique_by(.id))})
+             (($desired.models.providers[$e.key] // {}) * {models: (($e.value.models // []) | unique_by(.id))}) as $desiredProvider
+             | .models.providers[$e.key] = ((.models.providers[$e.key] // {}) * $desiredProvider)
            )
        else
          .
@@ -852,16 +882,29 @@ if [ -f "$EXISTING_CONFIG" ]; then
     "$EXISTING_CONFIG" 2>/dev/null)
 
   if [ -n "$PATCHED" ]; then
-    echo "$PATCHED" > "$EXISTING_CONFIG.tmp" \
-      && mv "$EXISTING_CONFIG.tmp" "$EXISTING_CONFIG"
-    echo "Config patched successfully."
+    backup_config_copy "$EXISTING_CONFIG"
+    if write_json_atomic "$EXISTING_CONFIG" "$PATCHED"; then
+      echo "Config patched successfully."
+    else
+      echo "Patch produced invalid JSON — writing fresh config."
+      write_json_atomic "$EXISTING_CONFIG" "$CONFIG_JSON" || { echo "ERROR: could not write valid fallback config" >&2; exit 1; }
+    fi
   else
-    echo "Patch failed — writing fresh config."
-    echo "$CONFIG_JSON" > "$EXISTING_CONFIG"
+    echo "Patch failed."
+    # Validate only on patch failure (as requested). If restored config is invalid,
+    # quarantine it and regenerate from runtime config; otherwise keep it untouched.
+    if ! validate_json_file "$EXISTING_CONFIG"; then
+      echo "Restored config is invalid JSON — backing up and regenerating from runtime config."
+      cp "$EXISTING_CONFIG" "${EXISTING_CONFIG}.invalid.$(date +%Y%m%d-%H%M%S)" 2>/dev/null || true
+      backup_config_copy "$EXISTING_CONFIG"
+      write_json_atomic "$EXISTING_CONFIG" "$CONFIG_JSON" || { echo "ERROR: could not write valid fallback config" >&2; exit 1; }
+    else
+      echo "Patch failed but restored config is valid — keeping existing config unchanged."
+    fi
   fi
 else
   echo "No restored config — writing fresh config..."
-  echo "$CONFIG_JSON" > "$EXISTING_CONFIG"
+  write_json_atomic "$EXISTING_CONFIG" "$CONFIG_JSON" || { echo "ERROR: could not write valid config" >&2; exit 1; }
 fi
 chmod 600 "$EXISTING_CONFIG"