File size: 1,510 Bytes
8d3471e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | 'use strict';
function buildUsage(prompt, thinking, output, outputTokens = 0, providedPromptTokens = 0) {
const reasoningTokens = estimateTokens(thinking);
const completionTokens = estimateTokens(output);
const finalPromptTokens = Number.isFinite(providedPromptTokens) && providedPromptTokens > 0 ? Math.trunc(providedPromptTokens) : estimateTokens(prompt);
const overriddenCompletionTokens = Number.isFinite(outputTokens) && outputTokens > 0 ? Math.trunc(outputTokens) : 0;
const finalCompletionTokens = overriddenCompletionTokens > 0 ? overriddenCompletionTokens : reasoningTokens + completionTokens;
return {
prompt_tokens: finalPromptTokens,
completion_tokens: finalCompletionTokens,
total_tokens: finalPromptTokens + finalCompletionTokens,
completion_tokens_details: {
reasoning_tokens: reasoningTokens,
},
};
}
function estimateTokens(text) {
const t = asTokenString(text);
if (!t) {
return 0;
}
let asciiChars = 0;
let nonASCIIChars = 0;
for (const ch of Array.from(t)) {
if (ch.charCodeAt(0) < 128) {
asciiChars += 1;
} else {
nonASCIIChars += 1;
}
}
const n = Math.floor(asciiChars / 4) + Math.floor((nonASCIIChars * 10 + 7) / 13);
return n < 1 ? 1 : n;
}
function asTokenString(v) {
if (typeof v === 'string') {
return v;
}
if (Array.isArray(v)) {
return asTokenString(v[0]);
}
if (v == null) {
return '';
}
return String(v);
}
module.exports = {
buildUsage,
estimateTokens,
};
|