import type { OpenClawConfig } from "../../config/config.js"; import type { AuthProfileFailureReason, AuthProfileStore, ProfileUsageStats } from "./types.js"; import { normalizeProviderId } from "../model-selection.js"; import { saveAuthProfileStore, updateAuthProfileStoreWithLock } from "./store.js"; function resolveProfileUnusableUntil(stats: ProfileUsageStats): number | null { const values = [stats.cooldownUntil, stats.disabledUntil] .filter((value): value is number => typeof value === "number") .filter((value) => Number.isFinite(value) && value > 0); if (values.length === 0) { return null; } return Math.max(...values); } /** * Check if a profile is currently in cooldown (due to rate limiting or errors). */ export function isProfileInCooldown(store: AuthProfileStore, profileId: string): boolean { const stats = store.usageStats?.[profileId]; if (!stats) { return false; } const unusableUntil = resolveProfileUnusableUntil(stats); return unusableUntil ? Date.now() < unusableUntil : false; } /** * Mark a profile as successfully used. Resets error count and updates lastUsed. * Uses store lock to avoid overwriting concurrent usage updates. */ export async function markAuthProfileUsed(params: { store: AuthProfileStore; profileId: string; agentDir?: string; }): Promise { const { store, profileId, agentDir } = params; const updated = await updateAuthProfileStoreWithLock({ agentDir, updater: (freshStore) => { if (!freshStore.profiles[profileId]) { return false; } freshStore.usageStats = freshStore.usageStats ?? {}; freshStore.usageStats[profileId] = { ...freshStore.usageStats[profileId], lastUsed: Date.now(), errorCount: 0, cooldownUntil: undefined, disabledUntil: undefined, disabledReason: undefined, failureCounts: undefined, }; return true; }, }); if (updated) { store.usageStats = updated.usageStats; return; } if (!store.profiles[profileId]) { return; } store.usageStats = store.usageStats ?? {}; store.usageStats[profileId] = { ...store.usageStats[profileId], lastUsed: Date.now(), errorCount: 0, cooldownUntil: undefined, disabledUntil: undefined, disabledReason: undefined, failureCounts: undefined, }; saveAuthProfileStore(store, agentDir); } export function calculateAuthProfileCooldownMs(errorCount: number): number { const normalized = Math.max(1, errorCount); return Math.min( 60 * 60 * 1000, // 1 hour max 60 * 1000 * 5 ** Math.min(normalized - 1, 3), ); } type ResolvedAuthCooldownConfig = { billingBackoffMs: number; billingMaxMs: number; failureWindowMs: number; }; function resolveAuthCooldownConfig(params: { cfg?: OpenClawConfig; providerId: string; }): ResolvedAuthCooldownConfig { const defaults = { billingBackoffHours: 5, billingMaxHours: 24, failureWindowHours: 24, } as const; const resolveHours = (value: unknown, fallback: number) => typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback; const cooldowns = params.cfg?.auth?.cooldowns; const billingOverride = (() => { const map = cooldowns?.billingBackoffHoursByProvider; if (!map) { return undefined; } for (const [key, value] of Object.entries(map)) { if (normalizeProviderId(key) === params.providerId) { return value; } } return undefined; })(); const billingBackoffHours = resolveHours( billingOverride ?? cooldowns?.billingBackoffHours, defaults.billingBackoffHours, ); const billingMaxHours = resolveHours(cooldowns?.billingMaxHours, defaults.billingMaxHours); const failureWindowHours = resolveHours( cooldowns?.failureWindowHours, defaults.failureWindowHours, ); return { billingBackoffMs: billingBackoffHours * 60 * 60 * 1000, billingMaxMs: billingMaxHours * 60 * 60 * 1000, failureWindowMs: failureWindowHours * 60 * 60 * 1000, }; } function calculateAuthProfileBillingDisableMsWithConfig(params: { errorCount: number; baseMs: number; maxMs: number; }): number { const normalized = Math.max(1, params.errorCount); const baseMs = Math.max(60_000, params.baseMs); const maxMs = Math.max(baseMs, params.maxMs); const exponent = Math.min(normalized - 1, 10); const raw = baseMs * 2 ** exponent; return Math.min(maxMs, raw); } export function resolveProfileUnusableUntilForDisplay( store: AuthProfileStore, profileId: string, ): number | null { const stats = store.usageStats?.[profileId]; if (!stats) { return null; } return resolveProfileUnusableUntil(stats); } function computeNextProfileUsageStats(params: { existing: ProfileUsageStats; now: number; reason: AuthProfileFailureReason; cfgResolved: ResolvedAuthCooldownConfig; }): ProfileUsageStats { const windowMs = params.cfgResolved.failureWindowMs; const windowExpired = typeof params.existing.lastFailureAt === "number" && params.existing.lastFailureAt > 0 && params.now - params.existing.lastFailureAt > windowMs; const baseErrorCount = windowExpired ? 0 : (params.existing.errorCount ?? 0); const nextErrorCount = baseErrorCount + 1; const failureCounts = windowExpired ? {} : { ...params.existing.failureCounts }; failureCounts[params.reason] = (failureCounts[params.reason] ?? 0) + 1; const updatedStats: ProfileUsageStats = { ...params.existing, errorCount: nextErrorCount, failureCounts, lastFailureAt: params.now, }; if (params.reason === "billing") { const billingCount = failureCounts.billing ?? 1; const backoffMs = calculateAuthProfileBillingDisableMsWithConfig({ errorCount: billingCount, baseMs: params.cfgResolved.billingBackoffMs, maxMs: params.cfgResolved.billingMaxMs, }); updatedStats.disabledUntil = params.now + backoffMs; updatedStats.disabledReason = "billing"; } else { const backoffMs = calculateAuthProfileCooldownMs(nextErrorCount); updatedStats.cooldownUntil = params.now + backoffMs; } return updatedStats; } /** * Mark a profile as failed for a specific reason. Billing failures are treated * as "disabled" (longer backoff) vs the regular cooldown window. */ export async function markAuthProfileFailure(params: { store: AuthProfileStore; profileId: string; reason: AuthProfileFailureReason; cfg?: OpenClawConfig; agentDir?: string; }): Promise { const { store, profileId, reason, agentDir, cfg } = params; const updated = await updateAuthProfileStoreWithLock({ agentDir, updater: (freshStore) => { const profile = freshStore.profiles[profileId]; if (!profile) { return false; } freshStore.usageStats = freshStore.usageStats ?? {}; const existing = freshStore.usageStats[profileId] ?? {}; const now = Date.now(); const providerKey = normalizeProviderId(profile.provider); const cfgResolved = resolveAuthCooldownConfig({ cfg, providerId: providerKey, }); freshStore.usageStats[profileId] = computeNextProfileUsageStats({ existing, now, reason, cfgResolved, }); return true; }, }); if (updated) { store.usageStats = updated.usageStats; return; } if (!store.profiles[profileId]) { return; } store.usageStats = store.usageStats ?? {}; const existing = store.usageStats[profileId] ?? {}; const now = Date.now(); const providerKey = normalizeProviderId(store.profiles[profileId]?.provider ?? ""); const cfgResolved = resolveAuthCooldownConfig({ cfg, providerId: providerKey, }); store.usageStats[profileId] = computeNextProfileUsageStats({ existing, now, reason, cfgResolved, }); saveAuthProfileStore(store, agentDir); } /** * Mark a profile as failed/rate-limited. Applies exponential backoff cooldown. * Cooldown times: 1min, 5min, 25min, max 1 hour. * Uses store lock to avoid overwriting concurrent usage updates. */ export async function markAuthProfileCooldown(params: { store: AuthProfileStore; profileId: string; agentDir?: string; }): Promise { await markAuthProfileFailure({ store: params.store, profileId: params.profileId, reason: "unknown", agentDir: params.agentDir, }); } /** * Clear cooldown for a profile (e.g., manual reset). * Uses store lock to avoid overwriting concurrent usage updates. */ export async function clearAuthProfileCooldown(params: { store: AuthProfileStore; profileId: string; agentDir?: string; }): Promise { const { store, profileId, agentDir } = params; const updated = await updateAuthProfileStoreWithLock({ agentDir, updater: (freshStore) => { if (!freshStore.usageStats?.[profileId]) { return false; } freshStore.usageStats[profileId] = { ...freshStore.usageStats[profileId], errorCount: 0, cooldownUntil: undefined, }; return true; }, }); if (updated) { store.usageStats = updated.usageStats; return; } if (!store.usageStats?.[profileId]) { return; } store.usageStats[profileId] = { ...store.usageStats[profileId], errorCount: 0, cooldownUntil: undefined, }; saveAuthProfileStore(store, agentDir); }