Spaces:
Running
Running
File size: 2,845 Bytes
bd28470 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 | import { logger } from "./logger";
interface BucketState {
tokens: number;
lastRefill: number;
}
/**
* Token bucket rate limiter per provider.
* Controls how many API calls can be made per time window.
*/
export class RateLimiter {
private buckets = new Map<string, BucketState>();
constructor(
private readonly maxTokens: number,
private readonly refillRateMs: number // how often to fully refill
) {}
/**
* Returns true if the call is allowed, false if rate limit exceeded.
*/
tryConsume(provider: string, tokens = 1): boolean {
const now = Date.now();
let bucket = this.buckets.get(provider);
if (!bucket) {
bucket = { tokens: this.maxTokens, lastRefill: now };
this.buckets.set(provider, bucket);
}
// Refill based on elapsed time
const elapsed = now - bucket.lastRefill;
if (elapsed >= this.refillRateMs) {
bucket.tokens = this.maxTokens;
bucket.lastRefill = now;
}
if (bucket.tokens < tokens) {
logger.warn({ provider, tokensLeft: bucket.tokens }, `[RateLimit] ${provider} throttled`);
return false;
}
bucket.tokens -= tokens;
return true;
}
/**
* Wait until a token is available (blocking version).
*/
async consume(provider: string, tokens = 1): Promise<void> {
while (!this.tryConsume(provider, tokens)) {
await new Promise((r) => setTimeout(r, 500));
}
}
}
// ─── Daily quota tracker (persisted in memory, resets at midnight) ────────
interface DailyQuota {
count: number;
date: string; // YYYY-MM-DD
}
const dailyQuotas = new Map<string, DailyQuota>();
function todayStr(): string {
return new Date().toISOString().split("T")[0];
}
export function checkDailyQuota(key: string, limit: number): boolean {
const today = todayStr();
const quota = dailyQuotas.get(key);
if (!quota || quota.date !== today) {
dailyQuotas.set(key, { count: 0, date: today });
return true;
}
if (quota.count >= limit) {
logger.warn({ key, count: quota.count, limit }, `[DailyQuota] ${key} limit reached`);
return false;
}
return true;
}
export function incrementDailyQuota(key: string): void {
const today = todayStr();
const quota = dailyQuotas.get(key) ?? { count: 0, date: today };
if (quota.date !== today) {
quota.count = 0;
quota.date = today;
}
quota.count += 1;
dailyQuotas.set(key, quota);
}
// Pre-configured limiters for each provider
export const serperLimiter = new RateLimiter(10, 60_000); // 10 req/min
export const hunterLimiter = new RateLimiter(5, 60_000); // 5 req/min
export const snovLimiter = new RateLimiter(5, 60_000); // 5 req/min
export const reoonLimiter = new RateLimiter(10, 60_000); // 10 req/min
export const playwrightLimiter = new RateLimiter(3, 10_000); // 3 pages per 10s
|