lib/src/limits.js

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.TokenLimits = void 0;
class TokenLimits {
    maxTokens;
    requestTokens;
    responseTokens;
    knowledgeCutOff;
    constructor(model = 'llama-3.1-8b-instant') {
        this.knowledgeCutOff = '2023-12-01';
        // Adjust for Groq TPM/RPM limits on Free/On-Demand tier
        if (model.includes('llama-4')) {
            this.maxTokens = 29500;
            this.responseTokens = 4000;
        }
        else if (model.includes('llama-3.1-8b') ||
            model.includes('llama-3.2-1') ||
            model.includes('llama-3.1-70b') ||
            model.includes('llama-3.3-70b')) {
            // Groq limits are ~6k TPM for 8b and ~12k TPM for 70b
            this.maxTokens = model.includes('8b') ? 5500 : 11500;
            this.responseTokens = 2000; // Reserve less for response to allow more context
        }
        else {
            this.maxTokens = 128000;
            this.responseTokens = 16000;
        }
        // Provide some margin
        this.requestTokens = this.maxTokens - this.responseTokens - 500;
    }
    string() {
        return `max_tokens=${this.maxTokens}, request_tokens=${this.requestTokens}, response_tokens=${this.responseTokens}`;
    }
}
exports.TokenLimits = TokenLimits;
//# sourceMappingURL=limits.js.map