tests/make_tokenizer_cases.py

#!/usr/bin/env python3
# Copyright 2026 Everpaper. Licensed under the Apache License, Version 2.0 (see ../LICENSE).
"""Generate the tokenizer conformance fixture `tokenizer-cases.json`: >= 3,000 deterministic, FICTIONAL texts that
stress the Gemma tokenizer (CRLF, NFD accents, emoji ZWJ / skin tones / flags / keycaps, Arabic with diacritics, Thai,
Hindi, CJK, whitespace runs > 31, tabs, control characters, special-token look-alikes, very long inputs), with the ids
produced by the Hugging Face `tokenizers` library for `<bos> + encode(prefix + text) + <eos>` (no truncation).

    python3 make_tokenizer_cases.py --tokenizer ../model/tokenizer/tokenizer.json --out tokenizer-cases.json

The people, companies, events and numbers below are invented. Most place names are invented too; a few real
place names and words appear in the accent and script stress strings (e.g. Łódź, Đà Nẵng, Ångström).
"""
from __future__ import annotations

import argparse
import json
import random
import unicodedata
from pathlib import Path

PREFIX = {"q": "task: search result | query: ", "d": "title: none | text: ", "raw": ""}

FIRST = ["Orla", "Teodor", "Mirela", "Kasimir", "Ysolde", "Bram", "Fenna", "Ruaridh", "Ottilie", "Severin", "Jolan",
         "Liesl", "Corvin", "Tamsin", "Ilse", "Quillon", "Saskia", "Elowen", "Thaddeus", "Perpetua"]
LAST = ["Vantrell", "Dunholm", "Brecklow", "Szarvas", "Penhallow", "Lindqvar", "Castelmoro", "Ravensby", "Quarterman",
        "Halloran-Bey", "Møllergaard", "Åkerlund", "Nowakówna", "Çelikkaya", "Đặng", "O'Rourke-Vale"]
ORGS = ["Lumenvale", "Brightwater Mills", "Okkam Freight", "Velmora Labs", "Quillfeather", "Saltmarsh & Co",
        "Norrow Analytics", "Cobaltine", "Haverly Bikes", "Studio Morello", "Tavola Verde", "Pinecrest Dental"]
TOWNS = ["Brindlemoor", "Castel Ferrino", "Ostwick", "Valdorna", "Pellham Cross", "Marrowgate", "San Lucetto",
         "Hadleigh Fen", "Rocca Bruna", "Thistleby", "Søndervik", "Łęczyca Nowa", "Çamlıdere", "Žďár Horní"]
THINGS = ["the quarterly invoice", "the boiler service", "the onboarding checklist", "the API rate limits",
          "the warehouse lease", "the beta feedback", "the dentist appointment", "the hiring plan", "the tax return",
          "the server migration", "the book club", "the roof estimate", "the ferry timetable"]
COSE = ["la fattura trimestrale", "la revisione della caldaia", "il contratto d'affitto", "il preventivo del tetto",
        "la riunione di condominio", "il volo per Valdorna", "la visita dal dentista", "il piano ferie",
        "il trasloco di sabato", "il corso di ceramica"]
DAYS = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
MONTHS = ["January", "March", "May", "July", "September", "October", "November", "December"]

EN_TPL = [
    "{p} from {o} moved {t} to {d} because the team in {town} is short-staffed this month.",
    "The budget for {t} is {n},{m:03d} and {p} wants a second quote before {d}.",
    "Remember to call {p} about {t}; the office in {town} closes at {h}:30.",
    "{o} signed the renewal: {n} seats at {m} per seat, billed every {mo}.",
    "On {d} the deploy failed twice because a config flag was read as seconds instead of milliseconds.",
    "Minutes: {p} presented the roadmap, {p2} raised concerns about {t}, and the decision was postponed to {mo}.",
    "Our p99 latency rose to {m} ms after the {d} release; rollback is ready if it passes {h}0 ms.",
    "Grocery list: oat milk, two lemons, basmati rice, coriander, and dog treats for Biscuit.",
    "Flight {o2}{n} to {town} departs at {h}:{m2:02d}; seat {m2}C, one checked bag.",
    "Draft reply to {p}: thanks for the notes on {t}, we will send the revised version by {d}.",
    "The landlord in {town} agreed to fix the window before {mo}; the rent stays at {n},{m:03d} a year.",
]
IT_TPL = [
    "{p} di {o} ha spostato {c} a {d} perché l'ufficio di {town} è sotto organico.",
    "Il preventivo per {c} è di {n}.{m:03d} euro e {p} vuole un secondo parere entro {d}.",
    "Ricordati di chiamare {p} per {c}: lo studio di {town} chiude alle {h}:30.",
    "Verbale: {p} ha presentato il piano, {p2} ha sollevato dubbi su {c} e la decisione slitta a {mo}.",
    "La caldaia perde ancora; il tecnico dice che la valvola costa {m} euro e serve una settimana.",
    "Lista della spesa: latte d'avena, due limoni, riso basmati, coriandolo e croccantini per il cane.",
    "Il volo per {town} parte alle {h}:{m2:02d}; posto {m2}C, un bagaglio in stiva. Perché così presto?",
]
OTHER = {
    "de": ["Die Heizung in {town} fällt seit {d} ständig aus; {p} ruft morgen den Installateur an.",
           "Größere Änderungen am Vertrag mit {o} müssen bis {mo} unterschrieben werden."],
    "fr": ["{p} a déplacé la réunion à {h} h parce que le train pour {town} était en retard.",
           "Le devis de {o} s'élève à {n} {m:03d} € ; l'échéance est fixée à {mo}."],
    "es": ["¿Quién se encarga de {t} en {o}? {p} dijo que el plazo termina el {d}.",
           "El alquiler en {town} sube a {n}.{m:03d} al año; ¡hay que renegociar antes de {mo}!"],
    "pt": ["{p} avisou que a entrega de {o} chega a {town} na {d}, por volta das {h}h.",
           "O orçamento para {t} ficou em {n}.{m:03d}; a decisão foi adiada para {mo}."],
    "nl": ["{p} belt {d} met {o} over {t}; het kantoor in {town} sluit om {h}:30.",
           "De huur in {town} blijft {n}.{m:03d} per jaar als het raam vóór {mo} gemaakt is."],
    "pl": ["{p} przeniósł spotkanie na {d}, bo zespół w {town} ma za mało ludzi.",
           "Faktura od {o} opiewa na {n} {m:03d} zł; termin płatności mija w {mo}."],
    "tr": ["{p}, {o} ile yapılan sözleşmeyi {mo} ayına kadar imzalamalı; ofis {h}:30'da kapanıyor.",
           "{town}'daki kira yılda {n}.{m:03d} olarak kalıyor; pencere {mo} öncesi tamir edilecek."],
    "vi": ["{p} đã dời cuộc họp sang {d} vì đội ở {town} đang thiếu người.",
           "Hóa đơn của {o} là {n}.{m:03d}; hạn thanh toán là tháng {m2}."],
    "ru": ["{p} перенёс встречу на {d}, потому что в {town} не хватает людей.",
           "Счёт от {o} на {n} {m:03d}; оплатить нужно до {mo}."],
    "el": ["Ο {p} μετέφερε τη συνάντηση την {d} επειδή η ομάδα στο {town} είναι λίγη.",
           "Το τιμολόγιο της {o} είναι {n}.{m:03d}· πληρωμή μέχρι {mo}."],
    "he": ["{p} הזיז את הפגישה ל-{d} כי הצוות ב-{town} קטן מדי.",
           "החשבונית מ-{o} היא {n},{m:03d}; לשלם עד {mo}."],
    "ar": ["نَقَلَ {p} الاجْتِماعَ إلى يَوْمِ {d} لِأَنَّ الفَريقَ في {town} قَليلُ العَدَدِ هذا الشَّهْرَ.",
           "الفاتُورَةُ مِنْ {o} بِقيمَةِ {n}٬{m:03d}؛ يَجِبُ الدَّفْعُ قَبْلَ {mo}.",
           "تَذَكَّرْ أَنْ تَتَّصِلَ بِـ{p} بِخُصُوصِ {t}؛ المَكْتَبُ يُغْلِقُ السّاعَةَ {h}:30."],
    "fa": ["{p} جلسه را به {d} موکول کرد چون تیم {town} این ماه کم‌نیرو است.",
           "صورت‌حساب {o} به مبلغ {n}٬{m:03d} است؛ پرداخت تا {mo}."],
    "th": ["{p} เลื่อนการประชุมไปวัน{d} เพราะทีมที่{town}มีคนไม่พอในเดือนนี้",
           "ใบแจ้งหนี้จาก {o} เป็นเงิน {n},{m:03d} บาท ต้องชำระก่อน{mo}",
           "อย่าลืมโทรหา{p}เรื่อง{t} สำนักงานปิดเวลา {h}:30 น."],
    "hi": ["{p} ने बैठक {d} को टाल दी क्योंकि {town} की टीम में इस महीने लोग कम हैं।",
           "{o} का चालान {n},{m:03d} रुपये का है; भुगतान {mo} से पहले करना है।",
           "{p} को {t} के बारे में फ़ोन करना याद रखें; दफ़्तर {h}:30 बजे बंद होता है।"],
    "bn": ["{p} সভাটি {d}-এ পিছিয়ে দিয়েছেন কারণ {town}-এর দলে এই মাসে লোক কম।",
           "{o}-এর চালান {n},{m:03d} টাকার; {mo}-এর আগে পরিশোধ করতে হবে।"],
    "ta": ["{p} கூட்டத்தை {d} அன்று ஒத்திவைத்தார்; {town} குழுவில் இந்த மாதம் ஆட்கள் குறைவு.",
           "{o} விலைப்பட்டியல் {n},{m:03d}; {mo}க்கு முன் செலுத்த வேண்டும்."],
    "zh": ["{p}把会议改到了{d}，因为{town}的团队这个月人手不够。",
           "{o}的发票金额是{n},{m:03d}元，必须在{mo}之前付款。",
           "记得给{p}打电话谈{t}；办公室{h}点半关门。"],
    "zh_tw": ["{p}把會議改到了{d}，因為{town}的團隊這個月人手不夠。",
              "{o}的發票金額是{n},{m:03d}元，必須在{mo}之前付款。"],
    "ja": ["{p}さんは{town}のチームが今月人手不足なので、会議を{d}に延期しました。",
           "{o}からの請求書は{n},{m:03d}円で、{mo}までに支払う必要があります。",
           "{t}について{p}さんに電話するのを忘れないで。オフィスは{h}時半に閉まります。"],
    "ko": ["{p}은(는) {town} 팀이 이번 달 인력이 부족해서 회의를 {d}로 미뤘습니다.",
           "{o}의 청구서는 {n},{m:03d}원이며 {mo} 전에 결제해야 합니다.",
           "{t} 건으로 {p}에게 전화하는 것 잊지 마세요. 사무실은 {h}시 30분에 닫습니다."],
    "ka": ["{p}-მა შეხვედრა {d}-ზე გადადო, რადგან {town}-ის გუნდი ამ თვეში მცირეა."],
    "hy": ["{p}-ը հանդիպումը տեղափոխեց {d}, քանի որ {town}-ի թիմը այս ամիս փոքր է։"],
    "am": ["{p} ስብሰባውን ወደ {d} አዛወረው፣ ምክንያቱም በ{town} ያለው ቡድን በዚህ ወር ትንሽ ነው።"],
    "uk": ["{p} переніс зустріч на {d}, бо команді в {town} бракує людей."],
    "cs": ["{p} přesunul schůzku na {d}, protože tým v {town} má tento měsíc málo lidí."],
    "fi": ["{p} siirsi kokouksen {d}, koska {town}n tiimi on tässä kuussa alimiehitetty."],
    "hu": ["{p} áthelyezte a megbeszélést {d}-ra, mert a {town}-i csapat ebben a hónapban kevés."],
    "sw": ["{p} alihamisha mkutano hadi {d} kwa sababu timu ya {town} ina watu wachache mwezi huu."],
    "id": ["{p} memindahkan rapat ke {d} karena tim di {town} kekurangan orang bulan ini."],
}
QUERIES = ["When is {t} due?", "Who handles {t} at {o}?", "How much did {o} quote for {t}?",
           "What did {p} decide about the {town} office?", "Which flight goes to {town} on {d}?",
           "Why was the deploy rolled back?", "What is the rent in {town}?", "Quando scade {c}?",
           "Chi si occupa di {c} a {town}?", "¿Cuánto cuesta {t}?", "Wann endet die Testphase von {o}?",
           "{town} 사무실 임대료는 얼마인가요?", "{o}の請求書はいつまで？", "{p}是谁？", "ما هو إيجار {town}؟",
           "{town}में किराया कितना है?", "ค่าเช่าใน{town}เท่าไหร่", "p99 latency", "SOC 2", "INC-{m}", "wifi password"]
CODE = [
    "def backoff(attempt: int, base: float = 0.5, cap: float = {m}.0) -> float:\n    return min(cap, base * 2 ** attempt)\n",
    "func retryDelay(_ attempt: Int) -> Duration {{\n\t.milliseconds(min({m}_000, 500 << attempt))\n}}\n",
    "SELECT customer_id, SUM(amount) AS total\nFROM invoices\nWHERE issued_at >= '2026-{m2:02d}-01'\nGROUP BY customer_id\nHAVING SUM(amount) > {n}{m};\n",
    "const queue = new Map();\nexport function enqueue(id, payload) {{\n  if (queue.size > {m}) throw new Error('queue full');\n  queue.set(id, {{ payload, tries: 0 }});\n}}\n",
    "#!/bin/sh\nset -eu\nfor f in logs/*.gz; do\n  zcat \"$f\" | grep -c 'status={h}0' || true\ndone\n",
    "{{\n  \"service\": \"{o2}-webhooks\",\n  \"region\": \"ams-1\",\n  \"retries\": {h},\n  \"timeout_ms\": {m}0\n}}\n",
    "class DeliveryQueue:\n    def __init__(self, limit={m}):\n        self.items = []\n\n    def push(self, item):\n        if len(self.items) >= self.limit:\n            raise OverflowError(item)\n",
    "fn parse_port(s: &str) -> Result<u16, String> {{\n    s.trim().parse::<u16>().map_err(|e| format!(\"bad port {{s}}: {{e}}\"))\n}}\n",
    "<div class=\"card\">\r\n\t<span id=\"x{m}\">&nbsp;</span>\r\n</div>\r\n",
    "x = [i**2 for i in range({m}) if i % {h} == 0]  # noqa\n\n\n\n",
]
EMOJI = ["👩‍💻", "👨‍👩‍👧‍👦", "🏳️‍🌈", "🏴‍☠️", "👍🏽", "🧑🏿‍🚀", "🇮🇹", "🇯🇵", "🇺🇳", "1️⃣", "#️⃣", "*️⃣", "❤️", "❤", "🫶🏼",
         "👩🏻‍❤️‍💋‍👨🏾", "🏃‍♀️", "🧘🏽‍♂️", "🫠", "🐈‍⬛", "🐻‍❄️", "😶‍🌫️", "🇿🇿", "🏴󠁧󠁢󠁳󠁣󠁴󠁿", "✌🏻", "☝🏿", "🅰️", "™", "©️", "⚠️"]
LOOKALIKES = ["<bos>", "<eos>", "<pad>", "<unk>", "<mask>", "<|tool>", "<tool|>", "<|tool_call>", "<tool_call|>",
              "<|tool_response>", "<tool_response|>", "<|\"|>", "<|think|>", "<|channel>", "<channel|>", "<|turn>",
              "<turn|>", "<|image>", "<|audio>", "<|image|>", "<|audio|>", "<image|>", "<audio|>", "<|video|>",
              "<start_of_turn>", "<end_of_turn>", "<bos", "bos>", "< bos >", "<BOS>", "<Bos>", "<eos><bos>", "<eos><eos>",
              "<|tool><tool|>", "<|tool", "<|tool>>", "<<bos>>", "<bos>text<eos>", "&lt;bos&gt;", "<0x0A>", "<0xFF>",
              "<unused0>", "[multimodal]", "<|im_start|>", "<s>", "</s>", "<|endoftext|>", "▁", "▁▁", "▁<bos>▁", "<▁bos>"]
CONTROLS = ["\x00", "\x01", "\x07", "\x08", "\x0b", "\x0c", "\x1b", "\x1f", "\x7f", "\x85", "\xa0", " ", " ",
            " ", "​", "‌", "‍", "‎", "‏", "
", "
", "‪", "‮", "⁠",
            "⁦", "⁩", "　", "", "￾", "￿", "�", "", "\U000e0001", "\U0010ffff",
            "\U0001d11e", "\U00020000", "\U0002a6d6", "\U0003134a"]
ACCENTED = ["crème brûlée à la façon de {town}", "Ñandú y pingüino en el Río Zañartú", "Søren Ångström, Þórður Ægir",
            "Łódź, Žilina, Ștefan, Țara", "ẞtraße, naïve, coöperate, résumé, exposé", "Việt Nam, Hồ Chí Minh? Đà Nẵng!",
            "Džemal, Ljubljana, Njegoš", "Ḥamid, Ṣalāḥ, Ṭāriq, Ẓāhir", "Ễ ô̂ ạ̈ ṓ ṩ ǟ", "Ά έ ή ί ό ύ ώ ϊ ϋ ΐ ΰ"]
WS = [" ", "\n", "\t", "\r\n", "\r", " \t", "　", "\xa0", "\n\n"]


def fill(tpl: str, r: random.Random) -> str:
    o = r.choice(ORGS)
    return tpl.format(p=f"{r.choice(FIRST)} {r.choice(LAST)}", p2=r.choice(FIRST), o=o, o2=o.split()[0].lower()[:6],
                      t=r.choice(THINGS), c=r.choice(COSE), d=r.choice(DAYS), mo=r.choice(MONTHS), town=r.choice(TOWNS),
                      n=r.randint(2, 98), m=r.randint(10, 999), m2=r.randint(1, 59), h=r.randint(7, 21))


def sentence(r: random.Random, langs=None) -> str:
    langs = langs or ["en", "en", "it"] + list(OTHER)
    lang = r.choice(langs)
    pool = {"en": EN_TPL, "it": IT_TPL}.get(lang) or OTHER[lang]
    return fill(r.choice(pool), r)


def rand_scalar(r: random.Random) -> str:
    """A random assigned-or-not scalar from a spread of blocks (never a surrogate)."""
    ranges = [(0x20, 0x7e), (0xa0, 0x24f), (0x370, 0x3ff), (0x400, 0x4ff), (0x590, 0x6ff), (0x900, 0x97f),
              (0xe00, 0xe7f), (0x1100, 0x11ff), (0x2000, 0x206f), (0x2190, 0x23ff), (0x2500, 0x27bf),
              (0x3000, 0x30ff), (0x4e00, 0x9fff), (0xac00, 0xd7a3), (0xe000, 0xf8ff), (0xfb00, 0xfeff),
              (0x10000, 0x1049f), (0x1f300, 0x1faff), (0x20000, 0x2a6df), (0xe0000, 0xe007f), (0x30000, 0x3134f),
              (0xf0000, 0xffffd), (0x100000, 0x10ffff)]
    lo, hi = r.choice(ranges)
    return chr(r.randint(lo, hi))


def build(r: random.Random) -> list[dict]:
    out: list[dict] = []

    def add(cat, text, kind=None):
        out.append({"cat": cat, "kind": kind or r.choice(["d", "q", "raw"]), "text": text})

    for _ in range(420):
        add("en", " ".join(sentence(r, ["en"]) for _ in range(r.randint(1, 4))), "d")
    for _ in range(220):
        add("it", " ".join(sentence(r, ["it"]) for _ in range(r.randint(1, 3))), "d")
    for lang in OTHER:
        for _ in range(14 if lang in ("ar", "th", "hi", "zh", "ja", "ko") else 7):
            add(lang, " ".join(sentence(r, [lang]) for _ in range(r.randint(1, 3))))
    for _ in range(300):
        add("query", fill(r.choice(QUERIES), r), "q")
    for _ in range(200):
        add("code", "".join(fill(r.choice(CODE), r) for _ in range(r.randint(1, 3))), "d")
    # CRLF and mixed line endings
    for _ in range(160):
        s = "\r\n".join(sentence(r) for _ in range(r.randint(1, 5)))
        s = r.choice([s, s + "\r\n", "\r\n" + s, s.replace("\r\n", "\r", 1), s.replace("\r\n", "\n\r", 1), s + "\r"])
        add("crlf", s)
    # NFD / NFKC / mixed normalization forms
    for _ in range(160):
        base = fill(r.choice(ACCENTED), r) + " " + sentence(r, ["it", "fr", "pl", "vi", "tr", "cs", "hu", "el", "ru"])
        form = r.choice(["NFD", "NFD", "NFKD", "NFKC", "NFC"])
        s = unicodedata.normalize(form, base)
        if r.random() < 0.4:  # mix NFC and NFD halves
            s = unicodedata.normalize("NFC", base[: len(base) // 2]) + unicodedata.normalize("NFD", base[len(base) // 2:])
        add("nfd", s)
    # emoji sequences
    for _ in range(220):
        parts = [r.choice(EMOJI) for _ in range(r.randint(1, 8))]
        words = sentence(r, ["en"]).split()
        s = " ".join(w + (r.choice(parts) if r.random() < 0.3 else "") for w in words) if r.random() < 0.5 else "".join(parts)
        add("emoji", s)
    # scripts with combining marks and no spaces
    for lang, n in (("ar", 100), ("fa", 40), ("he", 30), ("th", 110), ("hi", 100), ("bn", 40), ("ta", 40), ("zh", 70),
                    ("zh_tw", 30), ("ja", 70), ("ko", 70), ("ka", 20), ("hy", 20), ("am", 20)):
        for _ in range(n):
            s = sentence(r, [lang])
            if lang in ("ar", "fa") and r.random() < 0.5:
                s = "".join(c + (r.choice("ًٌٍَُِّْ") if r.random() < 0.3 and "ء" <= c <= "ي" else "") for c in s)
            if r.random() < 0.3:
                s = s + " " + sentence(r, ["en"])
            add(lang + "_x", s)
    # whitespace runs > 31, tabs, newline runs, mixed
    for _ in range(180):
        ws = r.choice(WS)
        k = r.choice([31, 32, 33, 40, 63, 64, 65, 100, 127, 128, 129, 200, 500])
        a, b = sentence(r, ["en", "it"]), sentence(r, ["en"])
        s = r.choice([a + ws * k + b, ws * k + a, a + ws * k, ws * k, a + " " * k + "\t" * (k // 3) + b, (ws * k + a) * 2])
        add("whitespace", s)
    for _ in range(100):
        s = "\t".join(sentence(r).split(" ")[: r.randint(1, 12)])
        s = r.choice([s, "\t" + s, s + "\t\t", s.replace("\t", "\t \t", 1), s.replace("\t", " \t ", 2)])
        add("tabs", s)
    # control characters and invisible code points
    for _ in range(120):
        words = sentence(r).split(" ")
        s = " ".join(w + (r.choice(CONTROLS) if r.random() < 0.35 else "") for w in words)
        s = r.choice([s, r.choice(CONTROLS) + s, s + r.choice(CONTROLS), "".join(r.choice(CONTROLS) for _ in range(r.randint(1, 12)))])
        add("control", s)
    # special-token look-alikes, alone, embedded, adjacent, repeated
    for _ in range(170):
        toks = [r.choice(LOOKALIKES) for _ in range(r.randint(1, 5))]
        words = sentence(r, ["en"]).split(" ")
        pos = r.randint(0, len(words))
        s = r.choice([" ".join(words[:pos] + toks + words[pos:]), "".join(toks), " ".join(toks),
                      "".join(words[:pos]) + "".join(toks) + "".join(words[pos:]), toks[0] * r.randint(2, 6)])
        add("lookalike", s)
    # very short / empty
    for s in ["", " ", "  ", "\n", "a", "é", "é", "▁", "▁a", "🙂", "<", ">", "|", "title:", "task:", "none", "x" * 1,
              "\u0000", "퟿", "", "\U0010ffff", "0", "٠١٢٣", "१२३", "一", "あ", "한"]:
        add("short", s, "raw")
        add("short", s, "d")
    # random scalar soup (assigned, unassigned, private use, non-characters)
    for _ in range(220):
        s = "".join(rand_scalar(r) for _ in range(r.randint(1, 60)))
        add("soup", s)
    # mixed everything
    for _ in range(200):
        bits = [sentence(r), r.choice(EMOJI), r.choice(LOOKALIKES), r.choice(CONTROLS), r.choice(WS) * r.randint(1, 40),
                fill(r.choice(CODE), r), unicodedata.normalize("NFD", fill(r.choice(ACCENTED), r)), "".join(rand_scalar(r) for _ in range(5))]
        r.shuffle(bits)
        add("mixed", "".join(bits[: r.randint(2, 8)]))
    # very long inputs (5k .. 80k characters)
    for i in range(22):
        n = [5000, 8000, 12000, 20000, 30000, 50000, 80000][i % 7]
        parts = []
        while sum(len(p) for p in parts) < n:
            parts.append(r.choice([sentence(r), fill(r.choice(CODE), r), r.choice(EMOJI), r.choice(WS) * r.randint(1, 50),
                                   "".join(rand_scalar(r) for _ in range(20)), r.choice(LOOKALIKES)]))
        add("long", " ".join(parts), "d")
    return out


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--tokenizer", required=True, help="path to tokenizer.json")
    ap.add_argument("--out", required=True)
    ap.add_argument("--seed", type=int, default=20261012)
    args = ap.parse_args()
    from tokenizers import Tokenizer

    tok = Tokenizer.from_file(args.tokenizer)
    cases = build(random.Random(args.seed))
    for c in cases:
        c["ids"] = [2] + tok.encode(PREFIX[c["kind"]] + c["text"], add_special_tokens=False).ids + [1]
    Path(args.out).write_text(json.dumps(cases, ensure_ascii=False), encoding="utf-8")
    from collections import Counter
    print(f"{len(cases)} cases, {sum(len(c['ids']) for c in cases)} ids ->", args.out)
    print(dict(Counter(c["cat"] for c in cases)))


if __name__ == "__main__":
    main()