tokenizer_info.json

{
  "repo": "Panhapich/khmer-sp-8k",
  "corpus_repo": "Panhapich/khmer-text-corpus",
  "vocab_size": 8000,
  "model_type": "unigram",
  "character_coverage": 0.9998,
  "special_tokens": {
    "PAD": {
      "piece": "<PAD>",
      "id": 0
    },
    "UNK": {
      "piece": "<UNK>",
      "id": 1
    },
    "BOS": {
      "piece": "<BOS>",
      "id": 2
    },
    "EOS": {
      "piece": "<EOS>",
      "id": 3
    },
    "MASK": {
      "piece": "<MASK>",
      "id": 4
    }
  },
  "files": [
    "khmer_sp.model",
    "khmer_sp.vocab",
    "khmer_segmentation.py",
    "gazetteer.json",
    "latin_exceptions.json"
  ],
  "requires_segmentation_pipeline": true,
  "segmentation_note": "This tokenizer was trained on khmer-nltk word-segmented text (with wordninja used to further decompose glued English/Latin runs, guarded by latin_exceptions.json). Always encode/decode through khmer_segmentation.KhmerTokenizer, not a bare SentencePieceProcessor, or cross-word token fusion reappears. See khmer_segmentation_sentencepiece_approach.md.",
  "trained_on_sources": [
    "Sokheng/khmer-synthetic-ocr-v1-100k",
    "rinabuoy/aupp-assignment-data",
    "nphearum/khmer-raw-text-3M-v2",
    "khmer_corpus.txt (local)"
  ],
  "created": "2026-07-07T18:32:04.965375Z"
}