reports/tokenizer.json

{
  "byte": {
    "english": {
      "characters": 77,
      "bytes": 77,
      "tokens": 77,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "hindi": {
      "characters": 72,
      "bytes": 188,
      "tokens": 188,
      "characters_per_token": 0.3829787234042553,
      "roundtrip": true
    },
    "hinglish": {
      "characters": 70,
      "bytes": 70,
      "tokens": 70,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "python": {
      "characters": 70,
      "bytes": 70,
      "tokens": 70,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "json": {
      "characters": 61,
      "bytes": 61,
      "tokens": 61,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "math": {
      "characters": 45,
      "bytes": 57,
      "tokens": 57,
      "characters_per_token": 0.7894736842105263,
      "roundtrip": true
    },
    "shell": {
      "characters": 38,
      "bytes": 38,
      "tokens": 38,
      "characters_per_token": 1.0,
      "roundtrip": true
    },
    "xml_url": {
      "characters": 66,
      "bytes": 67,
      "tokens": 67,
      "characters_per_token": 0.9850746268656716,
      "roundtrip": true
    }
  },
  "qwen_bpe": {
    "english": {
      "tokens": 13,
      "characters_per_token": 5.923076923076923,
      "roundtrip": true
    },
    "hindi": {
      "tokens": 61,
      "characters_per_token": 1.180327868852459,
      "roundtrip": true
    },
    "hinglish": {
      "tokens": 22,
      "characters_per_token": 3.1818181818181817,
      "roundtrip": true
    },
    "python": {
      "tokens": 18,
      "characters_per_token": 3.888888888888889,
      "roundtrip": true
    },
    "json": {
      "tokens": 14,
      "characters_per_token": 4.357142857142857,
      "roundtrip": true
    },
    "math": {
      "tokens": 33,
      "characters_per_token": 1.3636363636363635,
      "roundtrip": true
    },
    "shell": {
      "tokens": 11,
      "characters_per_token": 3.4545454545454546,
      "roundtrip": true
    },
    "xml_url": {
      "tokens": 21,
      "characters_per_token": 3.142857142857143,
      "roundtrip": true
    }
  },
  "limitation": "Eight public examples, not a representative benchmark or vocabulary-size study"
}