tokenizer_config.json

{
  "tokenizer_class": "IndicCanaryTokenizer",
  "layout": {
    "spl_tokens": [
      0,
      1152
    ],
    "multilingual": [
      1152,
      7152
    ]
  },
  "unk_id": 0,
  "nospeech_id": 1,
  "pad_id": 2,
  "eos_id": 3,
  "bos_id": 4,
  "prompt_langs": [
    "bn",
    "gu",
    "hi",
    "kn",
    "ml",
    "mr",
    "or",
    "ta",
    "te",
    "ur"
  ],
  "prompt_ids_by_lang": {
    "bn": [
      7,
      4,
      18,
      42,
      42,
      5,
      9,
      11,
      13,
      15
    ],
    "gu": [
      7,
      4,
      18,
      84,
      84,
      5,
      9,
      11,
      13,
      15
    ],
    "hi": [
      7,
      4,
      18,
      89,
      89,
      5,
      9,
      11,
      13,
      15
    ],
    "kn": [
      7,
      4,
      18,
      104,
      104,
      5,
      9,
      11,
      13,
      15
    ],
    "ml": [
      7,
      4,
      18,
      130,
      130,
      5,
      9,
      11,
      13,
      15
    ],
    "mr": [
      7,
      4,
      18,
      134,
      134,
      5,
      9,
      11,
      13,
      15
    ],
    "or": [
      7,
      4,
      18,
      149,
      149,
      5,
      9,
      11,
      13,
      15
    ],
    "ta": [
      7,
      4,
      18,
      185,
      185,
      5,
      9,
      11,
      13,
      15
    ],
    "te": [
      7,
      4,
      18,
      187,
      187,
      5,
      9,
      11,
      13,
      15
    ],
    "ur": [
      7,
      4,
      18,
      199,
      199,
      5,
      9,
      11,
      13,
      15
    ]
  }
}