|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import { AuthType } from '../core/contentGenerator.js'; |
|
|
|
|
|
export interface RetryOptions { |
|
|
maxAttempts: number; |
|
|
initialDelayMs: number; |
|
|
maxDelayMs: number; |
|
|
shouldRetry: (error: Error) => boolean; |
|
|
onPersistent429?: (authType?: string) => Promise<string | null>; |
|
|
authType?: string; |
|
|
} |
|
|
|
|
|
const DEFAULT_RETRY_OPTIONS: RetryOptions = { |
|
|
maxAttempts: 5, |
|
|
initialDelayMs: 5000, |
|
|
maxDelayMs: 30000, |
|
|
shouldRetry: defaultShouldRetry, |
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function defaultShouldRetry(error: Error | unknown): boolean { |
|
|
|
|
|
if (error && typeof (error as { status?: number }).status === 'number') { |
|
|
const status = (error as { status: number }).status; |
|
|
if (status === 429 || (status >= 500 && status < 600)) { |
|
|
return true; |
|
|
} |
|
|
} |
|
|
if (error instanceof Error && error.message) { |
|
|
if (error.message.includes('429')) return true; |
|
|
if (error.message.match(/5\d{2}/)) return true; |
|
|
} |
|
|
return false; |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function delay(ms: number): Promise<void> { |
|
|
return new Promise((resolve) => setTimeout(resolve, ms)); |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
export async function retryWithBackoff<T>( |
|
|
fn: () => Promise<T>, |
|
|
options?: Partial<RetryOptions>, |
|
|
): Promise<T> { |
|
|
const { |
|
|
maxAttempts, |
|
|
initialDelayMs, |
|
|
maxDelayMs, |
|
|
onPersistent429, |
|
|
authType, |
|
|
shouldRetry, |
|
|
} = { |
|
|
...DEFAULT_RETRY_OPTIONS, |
|
|
...options, |
|
|
}; |
|
|
|
|
|
let attempt = 0; |
|
|
let currentDelay = initialDelayMs; |
|
|
let consecutive429Count = 0; |
|
|
|
|
|
while (attempt < maxAttempts) { |
|
|
attempt++; |
|
|
try { |
|
|
return await fn(); |
|
|
} catch (error) { |
|
|
const errorStatus = getErrorStatus(error); |
|
|
|
|
|
|
|
|
if (errorStatus === 429) { |
|
|
consecutive429Count++; |
|
|
} else { |
|
|
consecutive429Count = 0; |
|
|
} |
|
|
|
|
|
|
|
|
if ( |
|
|
consecutive429Count >= 2 && |
|
|
onPersistent429 && |
|
|
authType === AuthType.LOGIN_WITH_GOOGLE_PERSONAL |
|
|
) { |
|
|
try { |
|
|
const fallbackModel = await onPersistent429(authType); |
|
|
if (fallbackModel) { |
|
|
|
|
|
attempt = 0; |
|
|
consecutive429Count = 0; |
|
|
currentDelay = initialDelayMs; |
|
|
|
|
|
continue; |
|
|
} |
|
|
} catch (fallbackError) { |
|
|
|
|
|
console.warn('Fallback to Flash model failed:', fallbackError); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
if (attempt >= maxAttempts || !shouldRetry(error as Error)) { |
|
|
throw error; |
|
|
} |
|
|
|
|
|
const { delayDurationMs, errorStatus: delayErrorStatus } = |
|
|
getDelayDurationAndStatus(error); |
|
|
|
|
|
if (delayDurationMs > 0) { |
|
|
|
|
|
console.warn( |
|
|
`Attempt ${attempt} failed with status ${delayErrorStatus ?? 'unknown'}. Retrying after explicit delay of ${delayDurationMs}ms...`, |
|
|
error, |
|
|
); |
|
|
await delay(delayDurationMs); |
|
|
|
|
|
currentDelay = initialDelayMs; |
|
|
} else { |
|
|
|
|
|
logRetryAttempt(attempt, error, errorStatus); |
|
|
|
|
|
const jitter = currentDelay * 0.3 * (Math.random() * 2 - 1); |
|
|
const delayWithJitter = Math.max(0, currentDelay + jitter); |
|
|
await delay(delayWithJitter); |
|
|
currentDelay = Math.min(maxDelayMs, currentDelay * 2); |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
throw new Error('Retry attempts exhausted'); |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function getErrorStatus(error: unknown): number | undefined { |
|
|
if (typeof error === 'object' && error !== null) { |
|
|
if ('status' in error && typeof error.status === 'number') { |
|
|
return error.status; |
|
|
} |
|
|
|
|
|
if ( |
|
|
'response' in error && |
|
|
typeof (error as { response?: unknown }).response === 'object' && |
|
|
(error as { response?: unknown }).response !== null |
|
|
) { |
|
|
const response = ( |
|
|
error as { response: { status?: unknown; headers?: unknown } } |
|
|
).response; |
|
|
if ('status' in response && typeof response.status === 'number') { |
|
|
return response.status; |
|
|
} |
|
|
} |
|
|
} |
|
|
return undefined; |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function getRetryAfterDelayMs(error: unknown): number { |
|
|
if (typeof error === 'object' && error !== null) { |
|
|
|
|
|
if ( |
|
|
'response' in error && |
|
|
typeof (error as { response?: unknown }).response === 'object' && |
|
|
(error as { response?: unknown }).response !== null |
|
|
) { |
|
|
const response = (error as { response: { headers?: unknown } }).response; |
|
|
if ( |
|
|
'headers' in response && |
|
|
typeof response.headers === 'object' && |
|
|
response.headers !== null |
|
|
) { |
|
|
const headers = response.headers as { 'retry-after'?: unknown }; |
|
|
const retryAfterHeader = headers['retry-after']; |
|
|
if (typeof retryAfterHeader === 'string') { |
|
|
const retryAfterSeconds = parseInt(retryAfterHeader, 10); |
|
|
if (!isNaN(retryAfterSeconds)) { |
|
|
return retryAfterSeconds * 1000; |
|
|
} |
|
|
|
|
|
const retryAfterDate = new Date(retryAfterHeader); |
|
|
if (!isNaN(retryAfterDate.getTime())) { |
|
|
return Math.max(0, retryAfterDate.getTime() - Date.now()); |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
return 0; |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function getDelayDurationAndStatus(error: unknown): { |
|
|
delayDurationMs: number; |
|
|
errorStatus: number | undefined; |
|
|
} { |
|
|
const errorStatus = getErrorStatus(error); |
|
|
let delayDurationMs = 0; |
|
|
|
|
|
if (errorStatus === 429) { |
|
|
delayDurationMs = getRetryAfterDelayMs(error); |
|
|
} |
|
|
return { delayDurationMs, errorStatus }; |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
function logRetryAttempt( |
|
|
attempt: number, |
|
|
error: unknown, |
|
|
errorStatus?: number, |
|
|
): void { |
|
|
let message = `Attempt ${attempt} failed. Retrying with backoff...`; |
|
|
if (errorStatus) { |
|
|
message = `Attempt ${attempt} failed with status ${errorStatus}. Retrying with backoff...`; |
|
|
} |
|
|
|
|
|
if (errorStatus === 429) { |
|
|
console.warn(message, error); |
|
|
} else if (errorStatus && errorStatus >= 500 && errorStatus < 600) { |
|
|
console.error(message, error); |
|
|
} else if (error instanceof Error) { |
|
|
|
|
|
if (error.message.includes('429')) { |
|
|
console.warn( |
|
|
`Attempt ${attempt} failed with 429 error (no Retry-After header). Retrying with backoff...`, |
|
|
error, |
|
|
); |
|
|
} else if (error.message.match(/5\d{2}/)) { |
|
|
console.error( |
|
|
`Attempt ${attempt} failed with 5xx error. Retrying with backoff...`, |
|
|
error, |
|
|
); |
|
|
} else { |
|
|
console.warn(message, error); |
|
|
} |
|
|
} else { |
|
|
console.warn(message, error); |
|
|
} |
|
|
} |
|
|
|