lib/src/services/token-scheduler.js

"use strict";
/**
 * Token Scheduler Service
 * Manages rate limiting for AI API calls
 *
 * NOTE: This is currently a per-process scheduler.
 * For horizontal scaling, replace with Redis-backed distributed rate limiter.
 */
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.heavyScheduler = exports.lightScheduler = exports.TokenScheduler = void 0;
const pino_1 = __importDefault(require("pino"));
const logger = (0, pino_1.default)({ level: process.env.LOG_LEVEL || 'info' });
class TokenScheduler {
    usedTokens = 0;
    limit = 10500; // Safety margin below 12k Groq TPM
    windowStart = Date.now();
    name;
    constructor(name = 'default') {
        this.name = name;
    }
    /**
     * Update the token limit
     */
    setLimit(newLimit) {
        this.limit = newLimit;
        logger.info(`[TokenScheduler:${this.name}] Limit updated to ${this.limit} tokens/min`);
    }
    /**
     * Get current metrics
     */
    getMetrics() {
        return {
            usedTokens: this.usedTokens,
            limit: this.limit,
            windowStart: this.windowStart
        };
    }
    /**
     * Wait for token budget to be available
     * Implements sliding window rate limiting
     */
    async wait(tokens) {
        const now = Date.now();
        // Reset window if 60 seconds have passed
        if (now - this.windowStart > 60000) {
            this.usedTokens = 0;
            this.windowStart = now;
        }
        // Check if we need to wait
        if (this.usedTokens + tokens > this.limit) {
            const waitTime = Math.max(0, 60000 - (now - this.windowStart) + 2000);
            logger.info(`[TokenScheduler:${this.name}] Budget exhausted (${this.usedTokens}/${this.limit}). Waiting ${Math.round(waitTime / 1000)}s...`);
            await new Promise(r => setTimeout(r, waitTime));
            this.usedTokens = 0; // Reset after waiting a full window
            this.windowStart = Date.now();
            return this.wait(tokens); // Recursive check
        }
        this.usedTokens += tokens;
    }
    /**
     * Reset the scheduler (useful for testing)
     */
    reset() {
        this.usedTokens = 0;
        this.windowStart = Date.now();
        logger.debug(`[TokenScheduler:${this.name}] Reset`);
    }
}
exports.TokenScheduler = TokenScheduler;
/**
 * Global scheduler instances (maintained for backward compatibility)
 *
 * TODO: Replace with Redis-backed distributed scheduler for horizontal scaling
 */
exports.lightScheduler = new TokenScheduler('light');
exports.heavyScheduler = new TokenScheduler('heavy');
//# sourceMappingURL=token-scheduler.js.map