tools/token-count-accuracy/test.js

/*

 * Copyright (C) 2024-present Puter Technologies Inc.

 * 

 * This file is part of Puter.

 * 

 * Puter is free software: you can redistribute it and/or modify

 * it under the terms of the GNU Affero General Public License as published

 * by the Free Software Foundation, either version 3 of the License, or

 * (at your option) any later version.

 * 

 * This program is distributed in the hope that it will be useful,

 * but WITHOUT ANY WARRANTY; without even the implied warranty of

 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

 * GNU Affero General Public License for more details.

 * 

 * You should have received a copy of the GNU Affero General Public License

 * along with this program.  If not, see <https://www.gnu.org/licenses/>.

 */

const claude_examples = [
    {
        type: 'output', // probably doesn't matter
        text: "I am Claude, an AI assistant created by Anthropic. I'm running on Puter, an open-source platform, through a driver interface called puter-chat-completion. I aim to be direct and honest about my identity and capabilities.",
        tokens: 55,
    },
    {
        type: 'output',
        text: `Here's a list of fascinating numbers and their significance:



1. 1.618033988749895 (φ, Phi, Golden Ratio)

- Found throughout nature and art

- Considered aesthetically pleasing

- Appears in the proportions of the Parthenon, nautilus shells, and spiral galaxies



2. 2.71828... (e, Euler's Number)

- Base of natural logarithms

- Fundamental to exponential growth and decay

- Essential in compound interest calculations



3. 3.14159... (π, Pi)

- Ratio of circle's circumference to diameter

- Transcendental number

- Appears across mathematics and physics



4. 6.02214076 × 10²³ (Avogadro's Number)

- Number of particles in one mole of substance

- Fundamental to chemistry

- Named after Italian scientist Amedeo Avogadro



5. 137 (Fine Structure Constant)

- Describes strength of electromagnetic interaction

- Mysterious number in physics

- No theoretical explanation for its value



6. 1729 (Hardy-Ramanujan Number)

- Smallest number expressible as sum of two cubes in two different ways

- Famous from conversation between mathematicians Hardy and Ramanujan



7. 42 

- Answer to "life, the universe, and everything" in Hitchhiker's Guide to the Galaxy

- Has gained cultural significance

- Interestingly appears in various mathematical contexts



8. 0.577215... (γ, Euler-Mascheroni Constant)

- Appears in number theory and calculus

- Still unknown if it's irrational

- Connected to harmonic series



9. 23! (Factorial of 23)

- Approximately equal to all seconds since Big Bang

- Shows how quickly factorial numbers grow



10. 666 (Number of the Beast)

- Cultural and religious significance

- Appears in Book of Revelation

- Subject of numerous mathematical curiosities



These numbers demonstrate how mathematics intersects with nature, science, culture, and human understanding of the universe.`,
        tokens: 481,
    },
    {
        type: 'output',
        text: `Here are some digits of Pi:

3.14159265358979323846264338327950288419716939937510...



Some digits of the square root of 2:

1.41421356237309504880168872420969807856967187537694...



And now for some random unpredictable things:



Purple elephants juggling quantum calculators in zero gravity

The sound of silence tastes like neon butterflies 

Yesterday's tomorrow forgot to attend its own birthday party

Clouds made of crystallized laughter are floating upside down

The number 7 decided to identify as a letter instead

Spinning teacups full of liquid starlight and abstract concepts

Time decided to flow sideways through a Klein bottle

Philosophical zombies debating the existence of consciousness while eating imaginary cookies

The color blue went on strike and was temporarily replaced by the smell of nostalgia

Dancing fractals wearing mismatched socks made of pure mathematics

A parade of impossible objects marching through an Escher painting

The concept of Tuesday learned to yodel in binary code

Metaphysical hiccups causing temporary glitches in the fabric of reason

Square circles plotting a revolution against euclidean geometry

The letter Q eloped with an ampersand and they had punctuation mark babies`,
        tokens: 284,
    }
];

// Measure each with tiktoken

class TikTokenCounter {
    constructor (model_to_try) {
        this.model_to_try = model_to_try;
    }

    get title () {
        return `TikToken ${this.model_to_try}`;
    }

    count (text) {
        const tiktoken = require('tiktoken');
        const enc = tiktoken.encoding_for_model(this.model_to_try);
        const tokens = enc.encode(text);
        return tokens.length;
    }
}

class DivideCounter {
    constructor (by) {
        this.by = by;
    }

    get title () {
        return `Divide by ${this.by}`;
    }

    count (text) {
        return text.length / this.by;
    }
}

const counters_to_try = [
    new TikTokenCounter('gpt-3.5-turbo'),
    new TikTokenCounter('gpt-4'),
    new TikTokenCounter('gpt-4o'),
    new TikTokenCounter('gpt-4o-mini'),
    new DivideCounter(4),
    new DivideCounter(5),
];

const scores = {};

const results = [];
for (const example of claude_examples) {
    const result = {
        example,
        counts: {},
        diffs: {},
    };
    for (const counter of counters_to_try) {
        result.counts[counter.title] = counter.count(example.text);
    }
    results.push(result);

    // Which one is the most accurate?
    const real_amount = example.tokens;
    for ( const count_name in result.counts ) {
        const count = result.counts[count_name];
        const diff = Math.abs(count - real_amount);
        result.diffs[count_name] = diff;
    }
    // Report the most accurate one
    const most_accurate =
        Object.keys(result.diffs)
            .reduce((a, b) => result.diffs[a] < result.diffs[b] ? a : b);
    result.most_accurate = most_accurate;

    scores[most_accurate] = (scores[most_accurate] || 0) + 1;
}


console.log(results);

console.log(scores);