Spaces:
Running
Running
File size: 5,808 Bytes
149698e c440d28 149698e c440d28 149698e c440d28 149698e ce1662b 149698e ce1662b 149698e 001455d 149698e 001455d 149698e 5252a42 f67c466 149698e 5252a42 149698e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 | import type { AIProvider } from '../services/aiService.js';
import type { ParsedTransaction } from '@icc/shared';
import pino from 'pino';
const logger = pino({ level: 'info' });
interface ProviderSlot {
provider: AIProvider;
cooldownUntil: number; // timestamp ms
dailyRequests: number;
dailyLimit: number;
disabled: boolean;
}
function sleep(ms: number) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export class AIProviderPool {
private slots: ProviderSlot[];
private currentIndex = 0;
constructor(providers: { provider: AIProvider; dailyLimit: number }[]) {
this.slots = providers.map(({ provider, dailyLimit }) => ({
provider,
cooldownUntil: 0,
dailyRequests: 0,
dailyLimit,
disabled: false,
}));
}
getCurrentSlotId(): string {
return this.slots[this.currentIndex]?.provider.name ?? 'none';
}
getStatus() {
return this.slots.map((slot) => ({
name: slot.provider.name,
available: !slot.disabled && Date.now() >= slot.cooldownUntil && slot.dailyRequests < slot.dailyLimit,
dailyRequests: slot.dailyRequests,
dailyLimit: slot.dailyLimit,
cooldownRemaining: Math.max(0, slot.cooldownUntil - Date.now()),
}));
}
private findAvailableSlot(): ProviderSlot | null {
const now = Date.now();
// Always start from slot 0 (primary provider) to enforce priority ordering
for (let i = 0; i < this.slots.length; i++) {
const slot = this.slots[i];
if (!slot.disabled && now >= slot.cooldownUntil && slot.dailyRequests < slot.dailyLimit) {
this.currentIndex = i;
return slot;
}
}
return null;
}
async parse(emailBody: string): Promise<ParsedTransaction> {
const maxRetries = this.slots.length * 3;
for (let attempt = 0; attempt < maxRetries; attempt++) {
const slot = this.findAvailableSlot();
if (!slot) {
// All slots exhausted — find the shortest cooldown and wait
const now = Date.now();
const soonest = this.slots
.filter((s) => !s.disabled && s.dailyRequests < s.dailyLimit)
.reduce((min, s) => Math.min(min, s.cooldownUntil), Infinity);
if (soonest === Infinity) {
throw new Error('All AI providers exhausted for the day');
}
const waitMs = Math.max(soonest - now, 1000);
logger.warn(`All providers rate-limited. Waiting ${Math.round(waitMs / 1000)}s...`);
await sleep(waitMs);
continue;
}
try {
logger.info(`Parsing with ${slot.provider.name} (attempt ${attempt + 1}/${maxRetries})`);
const result = await slot.provider.parse(emailBody);
slot.dailyRequests++;
return result;
} catch (error: any) {
if (error.status === 429 || error.status === 413) {
// Rate limited or request too large — set cooldown and try next
const retryAfter = parseInt(error.headers?.['retry-after'] || '30');
slot.cooldownUntil = Date.now() + retryAfter * 1000;
logger.info(
`${error.status === 413 ? 'Request too large' : 'Rate limited'} on ${slot.provider.name}, cooldown ${retryAfter}s. Switching...`
);
this.currentIndex = (this.currentIndex + 1) % this.slots.length;
continue;
}
if (error.status === 400) {
// 400 errors: invalid model, json_validate_failed, bad request
const bodyStr = error.body ?? error.message ?? '';
const isInvalidModel = bodyStr.includes('invalid_model') || bodyStr.includes('Invalid model');
if (isInvalidModel) {
slot.disabled = true;
logger.warn(`Invalid model on ${slot.provider.name}, permanently disabled`);
} else {
// Cooldown to prevent retry loop (findAvailableSlot starts from 0)
slot.cooldownUntil = Date.now() + 30_000;
logger.warn(`400 error on ${slot.provider.name}, 30s cooldown. Trying next...`);
}
this.currentIndex = (this.currentIndex + 1) % this.slots.length;
continue;
}
if (error.status === 403) {
// 403: model blocked at org level or forbidden — permanently disable
slot.disabled = true;
logger.warn(`Forbidden (403) on ${slot.provider.name}, permanently disabled. Trying next...`);
this.currentIndex = (this.currentIndex + 1) % this.slots.length;
continue;
}
if (error.status === 404) {
// 404: model does not exist on this provider — permanently disable the slot
slot.disabled = true;
logger.warn(`Model not found (404) on ${slot.provider.name}, permanently disabled. Trying next...`);
this.currentIndex = (this.currentIndex + 1) % this.slots.length;
continue;
}
if (error.status >= 500) {
// Server error — 1 min cooldown
slot.cooldownUntil = Date.now() + 60_000;
logger.warn(`Server error on ${slot.provider.name}, 60s cooldown`);
this.currentIndex = (this.currentIndex + 1) % this.slots.length;
continue;
}
// Any other error (JSON parse, network, etc.) — cooldown and try next
slot.cooldownUntil = Date.now() + 10_000;
logger.warn(`Unexpected error on ${slot.provider.name}: ${error.message || error.status}. Trying next...`);
this.currentIndex = (this.currentIndex + 1) % this.slots.length;
continue;
}
}
throw new Error('Max retries exceeded across all providers');
}
/** Reset daily counters (call at midnight UTC) */
resetDaily() {
for (const slot of this.slots) {
slot.dailyRequests = 0;
slot.disabled = false;
slot.cooldownUntil = 0;
}
}
}
|