tokenizer_audit.json

{
  "v2": {
    "vocab_size": 32000,
    "pollution_summary": {
      "handler_suffix": 57,
      "and_camel_class": 43,
      "long_camel": 3,
      "uppercase_blob": 462,
      "engineering_suffix": 11,
      "very_long_token": 28
    },
    "unique_polluted_count": 551,
    "polluted_examples": [
      [
        3114,
        "Handler"
      ],
      [
        24052,
        "ugmentationHumaineHandler"
      ],
      [
        23384,
        " DevOpsAndCloudInfrastructureHandler"
      ],
      [
        24695,
        "AndAnalyticsHandler"
      ],
      [
        23259,
        "dustrialEngineeringHandler"
      ],
      [
        23384,
        " DevOpsAndCloudInfrastructureHandler"
      ],
      [
        24420,
        "énarisationAndSt"
      ],
      [
        24054,
        " ImmobilierAndInvestissementHandler"
      ],
      [
        24926,
        " NeurosciencesAndSciences"
      ],
      [
        22133,
        " RésolutiondeConflitsAndMé"
      ],
      [
        23385,
        "LevelSystemsProgrammingHandler"
      ],
      [
        24934,
        "PrivacyAndComplianceHandler"
      ],
      [
        23860,
        "AndCorporateLawHandler"
      ],
      [
        8046,
        "GRES"
      ],
      [
        30173,
        "BNBUSDT"
      ],
      [
        12631,
        "ULTA"
      ],
      [
        18722,
        "IPORTS"
      ],
      [
        4626,
        "CRITIQUE"
      ],
      [
        23259,
        "dustrialEngineeringHandler"
      ],
      [
        8914,
        " Engineering"
      ],
      [
        25336,
        " DataEngineeringAndSQLAvancéHandler"
      ],
      [
        24424,
        "ScénarisationAndStorytellingHandler"
      ],
      [
        23252,
        " ManufacturingAndIn"
      ],
      [
        23384,
        " DevOpsAndCloudInfrastructureHandler"
      ],
      [
        23385,
        "LevelSystemsProgrammingHandler"
      ],
      [
        30904,
        "                              "
      ],
      [
        24054,
        " ImmobilierAndInvestissementHandler"
      ],
      [
        23678,
        " JournalismeAndInvestigationHandler"
      ]
    ],
    "fertility": [
      {
        "corpus": "EN",
        "chars": 545,
        "tokens": 150,
        "tokens_per_100_chars": 27.5
      },
      {
        "corpus": "FR",
        "chars": 761,
        "tokens": 184,
        "tokens_per_100_chars": 24.2
      },
      {
        "corpus": "code",
        "chars": 1332,
        "tokens": 585,
        "tokens_per_100_chars": 43.9
      },
      {
        "corpus": "tool_use_chatml",
        "chars": 477,
        "tokens": 182,
        "tokens_per_100_chars": 38.2
      }
    ],
    "fr_top_test": {
      "1_token": 140,
      "2_token": 23,
      "3_token": 20,
      "4plus_token": 7,
      "examples_high": [
        [
          "française",
          4
        ],
        [
          "chômage",
          4
        ],
        [
          "aujourd'hui",
          4
        ],
        [
          "qu'est-ce",
          5
        ],
        [
          "j'aime",
          4
        ],
        [
          "n'est-ce",
          5
        ],
        [
          "rejoindre",
          4
        ]
      ],
      "total": 190,
      "pct_1tok": 73.7,
      "pct_efficient": 85.8
    },
    "special_tokens": [
      [
        0,
        "<pad>",
        true
      ],
      [
        1,
        "<bos>",
        true
      ],
      [
        2,
        "<eos>",
        true
      ],
      [
        3,
        "<unk>",
        true
      ],
      [
        4,
        "<sep>",
        true
      ],
      [
        5,
        "<|teach|>",
        true
      ],
      [
        6,
        "<|evolve|>",
        true
      ],
      [
        7,
        "<|query|>",
        true
      ],
      [
        8,
        "<|code|>",
        true
      ],
      [
        9,
        "<|/code|>",
        true
      ],
      [
        10,
        "<|rust|>",
        true
      ],
      [
        11,
        "<|python|>",
        true
      ],
      [
        12,
        "<|system|>",
        true
      ],
      [
        13,
        "<|user|>",
        true
      ],
      [
        14,
        "<|archon|>",
        true
      ],
      [
        15,
        "<|think|>",
        true
      ]
    ]
  },
  "v21": {
    "vocab_size": 32006,
    "pollution_summary": {
      "handler_suffix": 57,
      "and_camel_class": 43,
      "long_camel": 3,
      "uppercase_blob": 462,
      "engineering_suffix": 11,
      "very_long_token": 28
    },
    "unique_polluted_count": 551,
    "polluted_examples": [
      [
        23176,
        "artupHandler"
      ],
      [
        24917,
        "tivesHandler"
      ],
      [
        24695,
        "AndAnalyticsHandler"
      ],
      [
        25501,
        " GénétiqueAndBiotechnologieHandler"
      ],
      [
        23850,
        "LawHandler"
      ],
      [
        22929,
        " ArchitectureAndUrbanisme"
      ],
      [
        23675,
        " JournalismeAndInvestigation"
      ],
      [
        24421,
        "ScénarisationAndSt"
      ],
      [
        22133,
        " RésolutiondeConflitsAndMé"
      ],
      [
        25501,
        " GénétiqueAndBiotechnologieHandler"
      ],
      [
        23860,
        "AndCorporateLawHandler"
      ],
      [
        23385,
        "LevelSystemsProgrammingHandler"
      ],
      [
        24934,
        "PrivacyAndComplianceHandler"
      ],
      [
        3315,
        "ORDRE"
      ],
      [
        15231,
        "VUSDT"
      ],
      [
        11028,
        "AUDE"
      ],
      [
        23399,
        "YOLO"
      ],
      [
        24086,
        "MPER"
      ],
      [
        23251,
        "dustrialEngineering"
      ],
      [
        11602,
        " Manufacturing"
      ],
      [
        23252,
        " ManufacturingAndIn"
      ],
      [
        15546,
        "Engineering"
      ],
      [
        8914,
        " Engineering"
      ],
      [
        8212,
        "|-------------|----------|------------"
      ],
      [
        25501,
        " GénétiqueAndBiotechnologieHandler"
      ],
      [
        24167,
        "ÃīnigmesAndRaisonnementLatéralHandler"
      ],
      [
        8229,
        "|-------------|----------|---------------|----------|"
      ],
      [
        30904,
        "                              "
      ]
    ],
    "fertility": [
      {
        "corpus": "EN",
        "chars": 545,
        "tokens": 150,
        "tokens_per_100_chars": 27.5
      },
      {
        "corpus": "FR",
        "chars": 761,
        "tokens": 184,
        "tokens_per_100_chars": 24.2
      },
      {
        "corpus": "code",
        "chars": 1332,
        "tokens": 585,
        "tokens_per_100_chars": 43.9
      },
      {
        "corpus": "tool_use_chatml",
        "chars": 477,
        "tokens": 141,
        "tokens_per_100_chars": 29.6
      }
    ],
    "fr_top_test": {
      "1_token": 140,
      "2_token": 23,
      "3_token": 20,
      "4plus_token": 7,
      "examples_high": [
        [
          "française",
          4
        ],
        [
          "chômage",
          4
        ],
        [
          "aujourd'hui",
          4
        ],
        [
          "qu'est-ce",
          5
        ],
        [
          "j'aime",
          4
        ],
        [
          "n'est-ce",
          5
        ],
        [
          "rejoindre",
          4
        ]
      ],
      "total": 190,
      "pct_1tok": 73.7,
      "pct_efficient": 85.8
    },
    "special_tokens": [
      [
        0,
        "<pad>",
        true
      ],
      [
        1,
        "<bos>",
        true
      ],
      [
        2,
        "<eos>",
        true
      ],
      [
        3,
        "<unk>",
        true
      ],
      [
        4,
        "<sep>",
        true
      ],
      [
        5,
        "<|teach|>",
        true
      ],
      [
        6,
        "<|evolve|>",
        true
      ],
      [
        7,
        "<|query|>",
        true
      ],
      [
        8,
        "<|code|>",
        true
      ],
      [
        9,
        "<|/code|>",
        true
      ],
      [
        10,
        "<|rust|>",
        true
      ],
      [
        11,
        "<|python|>",
        true
      ],
      [
        12,
        "<|system|>",
        true
      ],
      [
        13,
        "<|user|>",
        true
      ],
      [
        14,
        "<|archon|>",
        true
      ],
      [
        15,
        "<|think|>",
        true
      ],
      [
        32000,
        "<|im_start|>",
        true
      ],
      [
        32001,
        "<|im_end|>",
        true
      ],
      [
        32002,
        "<|assistant|>",
        true
      ],
      [
        32003,
        "<|tool_call|>",
        true
      ],
      [
        32004,
        "<|tool_result|>",
        true
      ],
      [
        32005,
        "<|task_type|>",
        true
      ]
    ]
  },
  "v21_diff": {
    "added_in_v21": [
      "<|assistant|>",
      "<|im_end|>",
      "<|im_start|>",
      "<|task_type|>",
      "<|tool_call|>",
      "<|tool_result|>"
    ],
    "removed_in_v21": []
  }
}