Hugging Face's logo

mkd-ai-dev
/
keural-tokenizer

Model card Files
xet
 Community
keural-tokenizer
File size: 3,593 Bytes
4c5d142
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
{
  "model_name": "Keural-13B Tokenizer",
  "status": "LOCKED - DO NOT MODIFY AFTER PRETRAINING",
  "created_at": "2026-03-11T11:55:40.906220",
  "corpus": {
    "file": "/home/work/keural-model/keural-model/data/raw/tokenizer_corpus_clean.txt",
    "size_gb": 26.74,
    "total_lines": 4345100,
    "total_chars": 25348127381,
    "estimated_tokens": 6337031845,
    "short_lines_lt_80": 0,
    "long_lines_gt_max_sentence_length": 422538
  },
  "tokenizer": {
    "model_type": "unigram",
    "vocab_size": 131072,
    "character_coverage": 0.9995,
    "byte_fallback": true,
    "split_digits": true,
    "split_by_unicode_script": true,
    "normalization_rule_name": "nfkc",
    "max_sentence_length": 10000,
    "num_threads": 32,
    "train_extremely_large_corpus": true,
    "shuffle_input_sentence": true
  },
  "special_tokens": {
    "pad": {
      "id": 0,
      "piece": "<pad>"
    },
    "bos": {
      "id": 1,
      "piece": "<bos>"
    },
    "eos": {
      "id": 2,
      "piece": "<eos>"
    },
    "unk": {
      "id": 3,
      "piece": "<unk>"
    }
  },
  "training": {
    "elapsed_minutes": 67.4,
    "train_log_file": "/home/work/keural-model/keural-model/data/logs/tokenizer_train_20260311_104736.log",
    "resource_log_file": "/home/work/keural-model/keural-model/data/logs/tokenizer_resources_20260311_104736.jsonl"
  },
  "validation": {
    "english_chars_per_token": 4.15,
    "validation_cases": {
      "korean": {
        "input": "안녕하세요 저는 큐럴 토크나이저를 테스트합니다.",
        "num_tokens": 10,
        "pieces_preview": [
          "안녕하세요",
          "▁저는",
          "▁큐",
          "럴",
          "▁토크",
          "나이",
          "저를",
          "▁테스트",
          "합니다",
          "."
        ],
        "roundtrip_ok": true
      },
      "english": {
        "input": "Hello, this is a tokenizer validation test for Keural.",
        "num_tokens": 13,
        "pieces_preview": [
          "Hello",
          ",",
          "▁this",
          "▁is",
          "▁a",
          "▁token",
          "izer",
          "▁validation",
          "▁test",
          "▁for",
          "▁Ke",
          "ural",
          "."
        ],
        "roundtrip_ok": true
      },
      "code": {
        "input": "def hello_world():\n    return 42",
        "num_tokens": 10,
        "pieces_preview": [
          "def",
          "▁hello",
          "_",
          "world",
          "():",
          "<0x0A>",
          "▁return",
          "▁",
          "4",
          "2"
        ],
        "roundtrip_ok": false
      },
      "mixed": {
        "input": "안녕하세요 world 123 def test(): return True",
        "num_tokens": 11,
        "pieces_preview": [
          "안녕하세요",
          "▁world",
          "▁",
          "1",
          "2",
          "3",
          "▁def",
          "▁test",
          "():",
          "▁return",
          "▁True"
        ],
        "roundtrip_ok": true
      },
      "rare_char": {
        "input": "𠜎",
        "num_tokens": 4,
        "pieces_preview": [
          "<0xF0>",
          "<0xA0>",
          "<0x9C>",
          "<0x8E>"
        ],
        "roundtrip_ok": true
      }
    }
  },
  "immutability": {
    "sha256_model": "b982818ea2f2057ba791e2006d17683799f1d8ceb9c91322018a638c4ec4b170",
    "model_file": "/home/work/keural-model/keural-model/tokenizer/keural_tokenizer.model",
    "vocab_file": "/home/work/keural-model/keural-model/tokenizer/keural_tokenizer.vocab"
  }
}