| { |
| "model_name": "Keural-13B Tokenizer", |
| "status": "LOCKED - DO NOT MODIFY AFTER PRETRAINING", |
| "created_at": "2026-03-11T11:55:40.906220", |
| "corpus": { |
| "file": "/home/work/keural-model/keural-model/data/raw/tokenizer_corpus_clean.txt", |
| "size_gb": 26.74, |
| "total_lines": 4345100, |
| "total_chars": 25348127381, |
| "estimated_tokens": 6337031845, |
| "short_lines_lt_80": 0, |
| "long_lines_gt_max_sentence_length": 422538 |
| }, |
| "tokenizer": { |
| "model_type": "unigram", |
| "vocab_size": 131072, |
| "character_coverage": 0.9995, |
| "byte_fallback": true, |
| "split_digits": true, |
| "split_by_unicode_script": true, |
| "normalization_rule_name": "nfkc", |
| "max_sentence_length": 10000, |
| "num_threads": 32, |
| "train_extremely_large_corpus": true, |
| "shuffle_input_sentence": true |
| }, |
| "special_tokens": { |
| "pad": { |
| "id": 0, |
| "piece": "<pad>" |
| }, |
| "bos": { |
| "id": 1, |
| "piece": "<bos>" |
| }, |
| "eos": { |
| "id": 2, |
| "piece": "<eos>" |
| }, |
| "unk": { |
| "id": 3, |
| "piece": "<unk>" |
| } |
| }, |
| "training": { |
| "elapsed_minutes": 67.4, |
| "train_log_file": "/home/work/keural-model/keural-model/data/logs/tokenizer_train_20260311_104736.log", |
| "resource_log_file": "/home/work/keural-model/keural-model/data/logs/tokenizer_resources_20260311_104736.jsonl" |
| }, |
| "validation": { |
| "english_chars_per_token": 4.15, |
| "validation_cases": { |
| "korean": { |
| "input": "안녕하세요 저는 큐럴 토크나이저를 테스트합니다.", |
| "num_tokens": 10, |
| "pieces_preview": [ |
| "안녕하세요", |
| "▁저는", |
| "▁큐", |
| "럴", |
| "▁토크", |
| "나이", |
| "저를", |
| "▁테스트", |
| "합니다", |
| "." |
| ], |
| "roundtrip_ok": true |
| }, |
| "english": { |
| "input": "Hello, this is a tokenizer validation test for Keural.", |
| "num_tokens": 13, |
| "pieces_preview": [ |
| "Hello", |
| ",", |
| "▁this", |
| "▁is", |
| "▁a", |
| "▁token", |
| "izer", |
| "▁validation", |
| "▁test", |
| "▁for", |
| "▁Ke", |
| "ural", |
| "." |
| ], |
| "roundtrip_ok": true |
| }, |
| "code": { |
| "input": "def hello_world():\n return 42", |
| "num_tokens": 10, |
| "pieces_preview": [ |
| "def", |
| "▁hello", |
| "_", |
| "world", |
| "():", |
| "<0x0A>", |
| "▁return", |
| "▁", |
| "4", |
| "2" |
| ], |
| "roundtrip_ok": false |
| }, |
| "mixed": { |
| "input": "안녕하세요 world 123 def test(): return True", |
| "num_tokens": 11, |
| "pieces_preview": [ |
| "안녕하세요", |
| "▁world", |
| "▁", |
| "1", |
| "2", |
| "3", |
| "▁def", |
| "▁test", |
| "():", |
| "▁return", |
| "▁True" |
| ], |
| "roundtrip_ok": true |
| }, |
| "rare_char": { |
| "input": "𠜎", |
| "num_tokens": 4, |
| "pieces_preview": [ |
| "<0xF0>", |
| "<0xA0>", |
| "<0x9C>", |
| "<0x8E>" |
| ], |
| "roundtrip_ok": true |
| } |
| } |
| }, |
| "immutability": { |
| "sha256_model": "b982818ea2f2057ba791e2006d17683799f1d8ceb9c91322018a638c4ec4b170", |
| "model_file": "/home/work/keural-model/keural-model/tokenizer/keural_tokenizer.model", |
| "vocab_file": "/home/work/keural-model/keural-model/tokenizer/keural_tokenizer.vocab" |
| } |
| } |
