File size: 7,311 Bytes
88c4c60 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 | import { ERROR_RULES, BACKOFF_CONFIG, TRANSIENT_COOLDOWN_MS } from "../config/errorConfig.js";
/**
* Calculate exponential backoff cooldown for rate limits (429)
* Level 1: 1s, Level 2: 2s, Level 3: 4s... → max 4 min
* @param {number} backoffLevel - Current backoff level
* @returns {number} Cooldown in milliseconds
*/
export function getQuotaCooldown(backoffLevel = 0) {
const level = Math.max(0, backoffLevel - 1);
const cooldown = BACKOFF_CONFIG.base * Math.pow(2, level);
return Math.min(cooldown, BACKOFF_CONFIG.max);
}
/**
* Check if error should trigger account fallback (switch to next account)
* Config-driven: matches ERROR_RULES top-to-bottom (text rules first, then status)
* @param {number} status - HTTP status code
* @param {string} errorText - Error message text
* @param {number} backoffLevel - Current backoff level for exponential backoff
* @returns {{ shouldFallback: boolean, cooldownMs: number, newBackoffLevel?: number }}
*/
export function checkFallbackError(status, errorText, backoffLevel = 0) {
const lowerError = errorText
? (typeof errorText === "string" ? errorText : JSON.stringify(errorText)).toLowerCase()
: "";
for (const rule of ERROR_RULES) {
// Text-based rule: match substring in error message
if (rule.text && lowerError && lowerError.includes(rule.text)) {
if (rule.backoff) {
const newLevel = Math.min(backoffLevel + 1, BACKOFF_CONFIG.maxLevel);
return { shouldFallback: true, cooldownMs: getQuotaCooldown(newLevel), newBackoffLevel: newLevel };
}
return { shouldFallback: true, cooldownMs: rule.cooldownMs };
}
// Status-based rule: match HTTP status code
if (rule.status && rule.status === status) {
if (rule.backoff) {
const newLevel = Math.min(backoffLevel + 1, BACKOFF_CONFIG.maxLevel);
return { shouldFallback: true, cooldownMs: getQuotaCooldown(newLevel), newBackoffLevel: newLevel };
}
return { shouldFallback: true, cooldownMs: rule.cooldownMs };
}
}
// Default: transient cooldown for any unmatched error
return { shouldFallback: true, cooldownMs: TRANSIENT_COOLDOWN_MS };
}
/**
* Check if account is currently unavailable (cooldown not expired)
*/
export function isAccountUnavailable(unavailableUntil) {
if (!unavailableUntil) return false;
return new Date(unavailableUntil).getTime() > Date.now();
}
/**
* Calculate unavailable until timestamp
*/
export function getUnavailableUntil(cooldownMs) {
return new Date(Date.now() + cooldownMs).toISOString();
}
/**
* Get the earliest rateLimitedUntil from a list of accounts
* @param {Array} accounts - Array of account objects with rateLimitedUntil
* @returns {string|null} Earliest rateLimitedUntil ISO string, or null
*/
export function getEarliestRateLimitedUntil(accounts) {
let earliest = null;
const now = Date.now();
for (const acc of accounts) {
if (!acc.rateLimitedUntil) continue;
const until = new Date(acc.rateLimitedUntil).getTime();
if (until <= now) continue;
if (!earliest || until < earliest) earliest = until;
}
if (!earliest) return null;
return new Date(earliest).toISOString();
}
/**
* Format rateLimitedUntil to human-readable "reset after Xm Ys"
* @param {string} rateLimitedUntil - ISO timestamp
* @returns {string} e.g. "reset after 2m 30s"
*/
export function formatRetryAfter(rateLimitedUntil) {
if (!rateLimitedUntil) return "";
const diffMs = new Date(rateLimitedUntil).getTime() - Date.now();
if (diffMs <= 0) return "reset after 0s";
const totalSec = Math.ceil(diffMs / 1000);
const h = Math.floor(totalSec / 3600);
const m = Math.floor((totalSec % 3600) / 60);
const s = totalSec % 60;
const parts = [];
if (h > 0) parts.push(`${h}h`);
if (m > 0) parts.push(`${m}m`);
if (s > 0 || parts.length === 0) parts.push(`${s}s`);
return `reset after ${parts.join(" ")}`;
}
/** Prefix for model lock flat fields on connection record */
export const MODEL_LOCK_PREFIX = "modelLock_";
/** Special key used when no model is known (account-level lock) */
export const MODEL_LOCK_ALL = `${MODEL_LOCK_PREFIX}__all`;
/** Build the flat field key for a model lock */
export function getModelLockKey(model) {
return model ? `${MODEL_LOCK_PREFIX}${model}` : MODEL_LOCK_ALL;
}
/**
* Check if a model lock on a connection is still active.
* Reads flat field `modelLock_${model}` (or `modelLock___all` when model=null).
*/
export function isModelLockActive(connection, model) {
const key = getModelLockKey(model);
const expiry = connection[key] || connection[MODEL_LOCK_ALL];
if (!expiry) return false;
return new Date(expiry).getTime() > Date.now();
}
/**
* Get earliest active model lock expiry across all modelLock_* fields.
* Used for UI cooldown display.
*/
export function getEarliestModelLockUntil(connection) {
if (!connection) return null;
let earliest = null;
const now = Date.now();
for (const [key, val] of Object.entries(connection)) {
if (!key.startsWith(MODEL_LOCK_PREFIX) || !val) continue;
const t = new Date(val).getTime();
if (t <= now) continue;
if (!earliest || t < earliest) earliest = t;
}
return earliest ? new Date(earliest).toISOString() : null;
}
/**
* Build update object to set a model lock on a connection.
*/
export function buildModelLockUpdate(model, cooldownMs) {
const key = getModelLockKey(model);
return { [key]: new Date(Date.now() + cooldownMs).toISOString() };
}
/**
* Build update object to clear all model locks on a connection.
*/
export function buildClearModelLocksUpdate(connection) {
const cleared = {};
for (const key of Object.keys(connection)) {
if (key.startsWith(MODEL_LOCK_PREFIX)) cleared[key] = null;
}
return cleared;
}
/**
* Filter available accounts (not in cooldown)
*/
export function filterAvailableAccounts(accounts, excludeId = null) {
const now = Date.now();
return accounts.filter(acc => {
if (excludeId && acc.id === excludeId) return false;
if (acc.rateLimitedUntil) {
const until = new Date(acc.rateLimitedUntil).getTime();
if (until > now) return false;
}
return true;
});
}
/**
* Reset account state when request succeeds
* Clears cooldown and resets backoff level to 0
* @param {object} account - Account object
* @returns {object} Updated account with reset state
*/
export function resetAccountState(account) {
if (!account) return account;
return {
...account,
rateLimitedUntil: null,
backoffLevel: 0,
lastError: null,
status: "active"
};
}
/**
* Apply error state to account
* @param {object} account - Account object
* @param {number} status - HTTP status code
* @param {string} errorText - Error message
* @returns {object} Updated account with error state
*/
export function applyErrorState(account, status, errorText) {
if (!account) return account;
const backoffLevel = account.backoffLevel || 0;
const { cooldownMs, newBackoffLevel } = checkFallbackError(status, errorText, backoffLevel);
return {
...account,
rateLimitedUntil: cooldownMs > 0 ? getUnavailableUntil(cooldownMs) : null,
backoffLevel: newBackoffLevel ?? backoffLevel,
lastError: { status, message: errorText, timestamp: new Date().toISOString() },
status: "error"
};
}
|