tokenizer/brahmic_init_map.json

{
  "description": "Brahmic cross-script embedding initialization map",
  "format": "new_token_id (str) → devanagari_token_id (int)",
  "original_vocab_size": 2602,
  "extended_vocab_size": 2871,
  "languages_added": [
    "bn",
    "ta",
    "ml",
    "mr",
    "gu"
  ],
  "n_mappings": 223,
  "map": {
    "2608": 1671,
    "2609": 1672,
    "2610": 1673,
    "2611": 1674,
    "2612": 1675,
    "2613": 1676,
    "2614": 1677,
    "2615": 1681,
    "2616": 1682,
    "2617": 1685,
    "2618": 1686,
    "2619": 1687,
    "2620": 1688,
    "2621": 1689,
    "2622": 1690,
    "2623": 1691,
    "2624": 1692,
    "2625": 1693,
    "2626": 1694,
    "2627": 1695,
    "2628": 1696,
    "2629": 1697,
    "2630": 1698,
    "2632": 1700,
    "2633": 1701,
    "2634": 1702,
    "2635": 1703,
    "2636": 1704,
    "2637": 1705,
    "2638": 1706,
    "2639": 1708,
    "2640": 1709,
    "2641": 1710,
    "2642": 1711,
    "2643": 1712,
    "2644": 1713,
    "2645": 1714,
    "2646": 1715,
    "2647": 1718,
    "2648": 1719,
    "2649": 1720,
    "2650": 1721,
    "2652": 1725,
    "2653": 1726,
    "2654": 1727,
    "2655": 1728,
    "2656": 1729,
    "2657": 1730,
    "2658": 1734,
    "2659": 1735,
    "2660": 1738,
    "2661": 1739,
    "2606": 1668,
    "2607": 1669,
    "2662": 1740,
    "2675": 1671,
    "2676": 1672,
    "2677": 1673,
    "2678": 1674,
    "2679": 1675,
    "2680": 1676,
    "2681": 1681,
    "2682": 1682,
    "2683": 1682,
    "2684": 1685,
    "2685": 1686,
    "2686": 1686,
    "2687": 1687,
    "2688": 1691,
    "2689": 1692,
    "2691": 1696,
    "2692": 1697,
    "2693": 1701,
    "2694": 1702,
    "2695": 1706,
    "2697": 1708,
    "2698": 1712,
    "2699": 1713,
    "2700": 1714,
    "2702": 1715,
    "2705": 1717,
    "2704": 1716,
    "2703": 1716,
    "2701": 1714,
    "2696": 1706,
    "2690": 1694,
    "2706": 1719,
    "2707": 1720,
    "2708": 1721,
    "2709": 1725,
    "2710": 1726,
    "2711": 1727,
    "2712": 1728,
    "2713": 1729,
    "2714": 1734,
    "2715": 1735,
    "2716": 1735,
    "2717": 1738,
    "2718": 1739,
    "2719": 1739,
    "2673": 1668,
    "2674": 1669,
    "2720": 1740,
    "2733": 1671,
    "2734": 1672,
    "2735": 1673,
    "2736": 1674,
    "2737": 1675,
    "2738": 1676,
    "2739": 1677,
    "2740": 1681,
    "2741": 1682,
    "2742": 1682,
    "2743": 1685,
    "2744": 1686,
    "2745": 1686,
    "2746": 1687,
    "2747": 1688,
    "2748": 1689,
    "2749": 1690,
    "2750": 1691,
    "2751": 1692,
    "2752": 1693,
    "2753": 1694,
    "2754": 1695,
    "2755": 1696,
    "2756": 1697,
    "2757": 1698,
    "2758": 1699,
    "2759": 1700,
    "2760": 1701,
    "2761": 1702,
    "2762": 1703,
    "2763": 1704,
    "2764": 1705,
    "2765": 1706,
    "2766": 1708,
    "2767": 1709,
    "2768": 1710,
    "2769": 1711,
    "2770": 1712,
    "2771": 1713,
    "2772": 1714,
    "2773": 1715,
    "2774": 1716,
    "2775": 1717,
    "2776": 1718,
    "2777": 1719,
    "2778": 1720,
    "2779": 1721,
    "2780": 1725,
    "2781": 1726,
    "2782": 1727,
    "2783": 1728,
    "2784": 1729,
    "2785": 1730,
    "2786": 1734,
    "2787": 1735,
    "2788": 1735,
    "2789": 1738,
    "2790": 1739,
    "2791": 1739,
    "2731": 1668,
    "2732": 1669,
    "2792": 1740,
    "2805": 1671,
    "2806": 1672,
    "2807": 1673,
    "2808": 1674,
    "2809": 1675,
    "2810": 1676,
    "2811": 1677,
    "2812": 1681,
    "2813": 1682,
    "2814": 1685,
    "2815": 1686,
    "2816": 1687,
    "2817": 1688,
    "2818": 1689,
    "2819": 1690,
    "2820": 1691,
    "2821": 1692,
    "2822": 1693,
    "2823": 1694,
    "2824": 1695,
    "2825": 1696,
    "2826": 1697,
    "2827": 1698,
    "2828": 1699,
    "2829": 1700,
    "2830": 1701,
    "2831": 1702,
    "2832": 1703,
    "2833": 1704,
    "2834": 1705,
    "2835": 1706,
    "2836": 1708,
    "2837": 1709,
    "2838": 1710,
    "2839": 1711,
    "2840": 1712,
    "2841": 1713,
    "2842": 1714,
    "2843": 1715,
    "2844": 1716,
    "2845": 1717,
    "2846": 1718,
    "2847": 1719,
    "2848": 1720,
    "2849": 1721,
    "2850": 1725,
    "2851": 1726,
    "2852": 1727,
    "2853": 1728,
    "2854": 1729,
    "2855": 1730,
    "2856": 1734,
    "2857": 1735,
    "2858": 1738,
    "2859": 1739,
    "2803": 1668,
    "2804": 1669,
    "2860": 1740
  },
  "readable": {
    "অ (bn)": "अ (hi) [id 1671→2608]",
    "আ (bn)": "आ (hi) [id 1672→2609]",
    "ই (bn)": "इ (hi) [id 1673→2610]",
    "ঈ (bn)": "ई (hi) [id 1674→2611]",
    "উ (bn)": "उ (hi) [id 1675→2612]",
    "ঊ (bn)": "ऊ (hi) [id 1676→2613]",
    "ঋ (bn)": "ऋ (hi) [id 1677→2614]",
    "এ (bn)": "ए (hi) [id 1681→2615]",
    "ঐ (bn)": "ऐ (hi) [id 1682→2616]",
    "ও (bn)": "ओ (hi) [id 1685→2617]",
    "ঔ (bn)": "औ (hi) [id 1686→2618]",
    "ক (bn)": "क (hi) [id 1687→2619]",
    "খ (bn)": "ख (hi) [id 1688→2620]",
    "গ (bn)": "ग (hi) [id 1689→2621]",
    "ঘ (bn)": "घ (hi) [id 1690→2622]",
    "ঙ (bn)": "ङ (hi) [id 1691→2623]",
    "চ (bn)": "च (hi) [id 1692→2624]",
    "ছ (bn)": "छ (hi) [id 1693→2625]",
    "জ (bn)": "ज (hi) [id 1694→2626]",
    "ঝ (bn)": "झ (hi) [id 1695→2627]",
    "ঞ (bn)": "ञ (hi) [id 1696→2628]",
    "ট (bn)": "ट (hi) [id 1697→2629]",
    "ঠ (bn)": "ठ (hi) [id 1698→2630]",
    "ঢ (bn)": "ढ (hi) [id 1700→2632]",
    "ণ (bn)": "ण (hi) [id 1701→2633]",
    "ত (bn)": "त (hi) [id 1702→2634]",
    "থ (bn)": "थ (hi) [id 1703→2635]",
    "দ (bn)": "द (hi) [id 1704→2636]",
    "ধ (bn)": "ध (hi) [id 1705→2637]",
    "ন (bn)": "न (hi) [id 1706→2638]",
    "প (bn)": "प (hi) [id 1708→2639]",
    "ফ (bn)": "फ (hi) [id 1709→2640]",
    "ব (bn)": "ब (hi) [id 1710→2641]",
    "ভ (bn)": "भ (hi) [id 1711→2642]",
    "ম (bn)": "म (hi) [id 1712→2643]",
    "য (bn)": "य (hi) [id 1713→2644]",
    "র (bn)": "र (hi) [id 1714→2645]",
    "ল (bn)": "ल (hi) [id 1715→2646]",
    "শ (bn)": "श (hi) [id 1718→2647]",
    "ষ (bn)": "ष (hi) [id 1719→2648]",
    "স (bn)": "स (hi) [id 1720→2649]",
    "হ (bn)": "ह (hi) [id 1721→2650]",
    "া (bn)": "ा (hi) [id 1725→2652]",
    "ি (bn)": "ि (hi) [id 1726→2653]",
    "ী (bn)": "ी (hi) [id 1727→2654]",
    "ু (bn)": "ु (hi) [id 1728→2655]",
    "ূ (bn)": "ू (hi) [id 1729→2656]",
    "ৃ (bn)": "ृ (hi) [id 1730→2657]",
    "ে (bn)": "े (hi) [id 1734→2658]",
    "ৈ (bn)": "ै (hi) [id 1735→2659]",
    "ো (bn)": "ो (hi) [id 1738→2660]",
    "ৌ (bn)": "ौ (hi) [id 1739→2661]",
    "ং (bn)": "ं (hi) [id 1668→2606]",
    "ঃ (bn)": "ः (hi) [id 1669→2607]",
    "্ (bn)": "् (hi) [id 1740→2662]",
    "அ (ta)": "अ (hi) [id 1671→2675]",
    "ஆ (ta)": "आ (hi) [id 1672→2676]",
    "இ (ta)": "इ (hi) [id 1673→2677]",
    "ஈ (ta)": "ई (hi) [id 1674→2678]",
    "உ (ta)": "उ (hi) [id 1675→2679]",
    "ஊ (ta)": "ऊ (hi) [id 1676→2680]",
    "எ (ta)": "ए (hi) [id 1681→2681]",
    "ஏ (ta)": "ऐ (hi) [id 1682→2682]",
    "ஐ (ta)": "ऐ (hi) [id 1682→2683]",
    "ஒ (ta)": "ओ (hi) [id 1685→2684]",
    "ஓ (ta)": "औ (hi) [id 1686→2685]",
    "ஔ (ta)": "औ (hi) [id 1686→2686]",
    "க (ta)": "क (hi) [id 1687→2687]",
    "ங (ta)": "ङ (hi) [id 1691→2688]",
    "ச (ta)": "च (hi) [id 1692→2689]",
    "ஞ (ta)": "ञ (hi) [id 1696→2691]",
    "ட (ta)": "ट (hi) [id 1697→2692]",
    "ண (ta)": "ण (hi) [id 1701→2693]",
    "த (ta)": "त (hi) [id 1702→2694]",
    "ந (ta)": "न (hi) [id 1706→2695]",
    "ப (ta)": "प (hi) [id 1708→2697]",
    "ம (ta)": "म (hi) [id 1712→2698]",
    "ய (ta)": "य (hi) [id 1713→2699]",
    "ர (ta)": "र (hi) [id 1714→2700]",
    "ல (ta)": "ल (hi) [id 1715→2702]",
    "வ (ta)": "व (hi) [id 1717→2705]",
    "ழ (ta)": "ळ (hi) [id 1716→2704]",
    "ள (ta)": "ळ (hi) [id 1716→2703]",
    "ற (ta)": "र (hi) [id 1714→2701]",
    "ன (ta)": "न (hi) [id 1706→2696]",
    "ஜ (ta)": "ज (hi) [id 1694→2690]",
    "ஷ (ta)": "ष (hi) [id 1719→2706]",
    "ஸ (ta)": "स (hi) [id 1720→2707]",
    "ஹ (ta)": "ह (hi) [id 1721→2708]",
    "ா (ta)": "ा (hi) [id 1725→2709]",
    "ி (ta)": "ि (hi) [id 1726→2710]",
    "ீ (ta)": "ी (hi) [id 1727→2711]",
    "ு (ta)": "ु (hi) [id 1728→2712]",
    "ூ (ta)": "ू (hi) [id 1729→2713]",
    "ெ (ta)": "े (hi) [id 1734→2714]",
    "ே (ta)": "ै (hi) [id 1735→2715]",
    "ை (ta)": "ै (hi) [id 1735→2716]",
    "ொ (ta)": "ो (hi) [id 1738→2717]",
    "ோ (ta)": "ौ (hi) [id 1739→2718]",
    "ௌ (ta)": "ौ (hi) [id 1739→2719]",
    "ஂ (ta)": "ं (hi) [id 1668→2673]",
    "ஃ (ta)": "ः (hi) [id 1669→2674]",
    "் (ta)": "् (hi) [id 1740→2720]",
    "അ (ml)": "अ (hi) [id 1671→2733]",
    "ആ (ml)": "आ (hi) [id 1672→2734]",
    "ഇ (ml)": "इ (hi) [id 1673→2735]",
    "ഈ (ml)": "ई (hi) [id 1674→2736]",
    "ഉ (ml)": "उ (hi) [id 1675→2737]",
    "ഊ (ml)": "ऊ (hi) [id 1676→2738]",
    "ഋ (ml)": "ऋ (hi) [id 1677→2739]",
    "എ (ml)": "ए (hi) [id 1681→2740]",
    "ഏ (ml)": "ऐ (hi) [id 1682→2741]",
    "ഐ (ml)": "ऐ (hi) [id 1682→2742]",
    "ഒ (ml)": "ओ (hi) [id 1685→2743]",
    "ഓ (ml)": "औ (hi) [id 1686→2744]",
    "ഔ (ml)": "औ (hi) [id 1686→2745]",
    "ക (ml)": "क (hi) [id 1687→2746]",
    "ഖ (ml)": "ख (hi) [id 1688→2747]",
    "ഗ (ml)": "ग (hi) [id 1689→2748]",
    "ഘ (ml)": "घ (hi) [id 1690→2749]",
    "ങ (ml)": "ङ (hi) [id 1691→2750]",
    "ച (ml)": "च (hi) [id 1692→2751]",
    "ഛ (ml)": "छ (hi) [id 1693→2752]",
    "ജ (ml)": "ज (hi) [id 1694→2753]",
    "ഝ (ml)": "झ (hi) [id 1695→2754]",
    "ഞ (ml)": "ञ (hi) [id 1696→2755]",
    "ട (ml)": "ट (hi) [id 1697→2756]",
    "ഠ (ml)": "ठ (hi) [id 1698→2757]",
    "ഡ (ml)": "ड (hi) [id 1699→2758]",
    "ഢ (ml)": "ढ (hi) [id 1700→2759]",
    "ണ (ml)": "ण (hi) [id 1701→2760]",
    "ത (ml)": "त (hi) [id 1702→2761]",
    "ഥ (ml)": "थ (hi) [id 1703→2762]",
    "ദ (ml)": "द (hi) [id 1704→2763]",
    "ധ (ml)": "ध (hi) [id 1705→2764]",
    "ന (ml)": "न (hi) [id 1706→2765]",
    "പ (ml)": "प (hi) [id 1708→2766]",
    "ഫ (ml)": "फ (hi) [id 1709→2767]",
    "ബ (ml)": "ब (hi) [id 1710→2768]",
    "ഭ (ml)": "भ (hi) [id 1711→2769]",
    "മ (ml)": "म (hi) [id 1712→2770]",
    "യ (ml)": "य (hi) [id 1713→2771]",
    "ര (ml)": "र (hi) [id 1714→2772]",
    "ല (ml)": "ल (hi) [id 1715→2773]",
    "ള (ml)": "ळ (hi) [id 1716→2774]",
    "വ (ml)": "व (hi) [id 1717→2775]",
    "ശ (ml)": "श (hi) [id 1718→2776]",
    "ഷ (ml)": "ष (hi) [id 1719→2777]",
    "സ (ml)": "स (hi) [id 1720→2778]",
    "ഹ (ml)": "ह (hi) [id 1721→2779]",
    "ാ (ml)": "ा (hi) [id 1725→2780]",
    "ി (ml)": "ि (hi) [id 1726→2781]",
    "ീ (ml)": "ी (hi) [id 1727→2782]",
    "ു (ml)": "ु (hi) [id 1728→2783]",
    "ൂ (ml)": "ू (hi) [id 1729→2784]",
    "ൃ (ml)": "ृ (hi) [id 1730→2785]",
    "െ (ml)": "े (hi) [id 1734→2786]",
    "േ (ml)": "ै (hi) [id 1735→2787]",
    "ൈ (ml)": "ै (hi) [id 1735→2788]",
    "ൊ (ml)": "ो (hi) [id 1738→2789]",
    "ോ (ml)": "ौ (hi) [id 1739→2790]",
    "ൌ (ml)": "ौ (hi) [id 1739→2791]",
    "ം (ml)": "ं (hi) [id 1668→2731]",
    "ഃ (ml)": "ः (hi) [id 1669→2732]",
    "് (ml)": "् (hi) [id 1740→2792]",
    "અ (gu)": "अ (hi) [id 1671→2805]",
    "આ (gu)": "आ (hi) [id 1672→2806]",
    "ઇ (gu)": "इ (hi) [id 1673→2807]",
    "ઈ (gu)": "ई (hi) [id 1674→2808]",
    "ઉ (gu)": "उ (hi) [id 1675→2809]",
    "ઊ (gu)": "ऊ (hi) [id 1676→2810]",
    "ઋ (gu)": "ऋ (hi) [id 1677→2811]",
    "એ (gu)": "ए (hi) [id 1681→2812]",
    "ઐ (gu)": "ऐ (hi) [id 1682→2813]",
    "ઓ (gu)": "ओ (hi) [id 1685→2814]",
    "ઔ (gu)": "औ (hi) [id 1686→2815]",
    "ક (gu)": "क (hi) [id 1687→2816]",
    "ખ (gu)": "ख (hi) [id 1688→2817]",
    "ગ (gu)": "ग (hi) [id 1689→2818]",
    "ઘ (gu)": "घ (hi) [id 1690→2819]",
    "ઙ (gu)": "ङ (hi) [id 1691→2820]",
    "ચ (gu)": "च (hi) [id 1692→2821]",
    "છ (gu)": "छ (hi) [id 1693→2822]",
    "જ (gu)": "ज (hi) [id 1694→2823]",
    "ઝ (gu)": "झ (hi) [id 1695→2824]",
    "ઞ (gu)": "ञ (hi) [id 1696→2825]",
    "ટ (gu)": "ट (hi) [id 1697→2826]",
    "ઠ (gu)": "ठ (hi) [id 1698→2827]",
    "ડ (gu)": "ड (hi) [id 1699→2828]",
    "ઢ (gu)": "ढ (hi) [id 1700→2829]",
    "ણ (gu)": "ण (hi) [id 1701→2830]",
    "ત (gu)": "त (hi) [id 1702→2831]",
    "થ (gu)": "थ (hi) [id 1703→2832]",
    "દ (gu)": "द (hi) [id 1704→2833]",
    "ધ (gu)": "ध (hi) [id 1705→2834]",
    "ન (gu)": "न (hi) [id 1706→2835]",
    "પ (gu)": "प (hi) [id 1708→2836]",
    "ફ (gu)": "फ (hi) [id 1709→2837]",
    "બ (gu)": "ब (hi) [id 1710→2838]",
    "ભ (gu)": "भ (hi) [id 1711→2839]",
    "મ (gu)": "म (hi) [id 1712→2840]",
    "ય (gu)": "य (hi) [id 1713→2841]",
    "ર (gu)": "र (hi) [id 1714→2842]",
    "લ (gu)": "ल (hi) [id 1715→2843]",
    "ળ (gu)": "ळ (hi) [id 1716→2844]",
    "વ (gu)": "व (hi) [id 1717→2845]",
    "શ (gu)": "श (hi) [id 1718→2846]",
    "ષ (gu)": "ष (hi) [id 1719→2847]",
    "સ (gu)": "स (hi) [id 1720→2848]",
    "હ (gu)": "ह (hi) [id 1721→2849]",
    "ા (gu)": "ा (hi) [id 1725→2850]",
    "િ (gu)": "ि (hi) [id 1726→2851]",
    "ી (gu)": "ी (hi) [id 1727→2852]",
    "ુ (gu)": "ु (hi) [id 1728→2853]",
    "ૂ (gu)": "ू (hi) [id 1729→2854]",
    "ૃ (gu)": "ृ (hi) [id 1730→2855]",
    "ે (gu)": "े (hi) [id 1734→2856]",
    "ૈ (gu)": "ै (hi) [id 1735→2857]",
    "ો (gu)": "ो (hi) [id 1738→2858]",
    "ૌ (gu)": "ौ (hi) [id 1739→2859]",
    "ં (gu)": "ं (hi) [id 1668→2803]",
    "ઃ (gu)": "ः (hi) [id 1669→2804]",
    "્ (gu)": "् (hi) [id 1740→2860]"
  }
}