Premier modèle d'architecture gemma from scratch

README.md

@@ -0,0 +1,54 @@
+---
+library_name: transformers
+tags:
+- generated_from_trainer
+model-index:
+- name: Nephaella
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# Nephaella
+
+This model is a fine-tuned version of [](https://huggingface.co/) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0005
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: linear
+- num_epochs: 4
+- mixed_precision_training: Native AMP
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 5.0.0
+- Pytorch 2.10.0+cpu
+- Datasets 4.0.0
+- Tokenizers 0.22.2

tokenizer.json

@@ -0,0 +1,2064 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "<mask>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 5,
+      "content": "प्रश्न:",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 6,
+      "content": "उत्तर:",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    }
+  ],
+  "normalizer": {
+    "type": "Replace",
+    "pattern": {
+      "String": " "
+    },
+    "content": "▁"
+  },
+  "pre_tokenizer": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {}
+  },
+  "decoder": {
+    "type": "Sequence",
+    "decoders": [
+      {
+        "type": "Replace",
+        "pattern": {
+          "String": "▁"
+        },
+        "content": " "
+      },
+      {
+        "type": "ByteFallback"
+      },
+      {
+        "type": "Fuse"
+      }
+    ]
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "<unk>",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": true,
+    "byte_fallback": true,
+    "ignore_merges": false,
+    "vocab": {
+      "<pad>": 0,
+      "</s>": 1,
+      "<s>": 2,
+      "<unk>": 3,
+      "<mask>": 4,
+      "प्रश्न:": 5,
+      "उत्तर:": 6,
+      "▁क": 7,
+      "▁प": 8,
+      "▁ह": 9,
+      "▁स": 10,
+      "▁म": 11,
+      "▁है": 12,
+      "्र": 13,
+      "र्": 14,
+      "या": 15,
+      "ें": 16,
+      "▁के": 17,
+      "▁ज": 18,
+      "▁ब": 19,
+      "▁ए": 20,
+      "ता": 21,
+      "▁प्र": 22,
+      "िक": 23,
+      "▁में": 24,
+      "▁उ": 25,
+      "ने": 26,
+      "▁अ": 27,
+      "▁कर": 28,
+      "▁का": 29,
+      "ों": 30,
+      "ना": 31,
+      "▁व": 32,
+      "▁ल": 33,
+      "▁पर": 34,
+      "▁कि": 35,
+      "▁हैं": 36,
+      "▁औ": 37,
+      "▁और": 38,
+      "लए": 39,
+      "ित": 40,
+      "▁द": 41,
+      "▁को": 42,
+      "ार": 43,
+      "क्": 44,
+      "▁त": 45,
+      "▁एक": 46,
+      "▁आ": 47,
+      "▁ग": 48,
+      "▁की": 49,
+      "ॉड": 50,
+      "▁(": 51,
+      "▁मॉड": 52,
+      "से": 53,
+      "्य": 54,
+      "▁[": 55,
+      "▁ट": 56,
+      "▁सं": 57,
+      "▁से": 58,
+      "▁मॉडल": 59,
+      "मा": 60,
+      "▁किया": 61,
+      "▁भ": 62,
+      "▁र": 63,
+      "ते": 64,
+      "▁न": 65,
+      "▁श": 66,
+      "कन": 67,
+      "ां": 68,
+      "▁लि": 69,
+      "स्": 70,
+      "ड़": 71,
+      "▁जा": 72,
+      "्ष": 73,
+      "▁लिए": 74,
+      "ोग": 75,
+      "लएम": 76,
+      "्या": 77,
+      "▁\"": 78,
+      "ाष": 79,
+      "▁उप": 80,
+      "▁एलए": 81,
+      "▁एलएलएम": 82,
+      "त्": 83,
+      "▁ड": 84,
+      "योग": 85,
+      "ाषा": 86,
+      "ोकन": 87,
+      "▁जो": 88,
+      "▁सा": 89,
+      "▁भाषा": 90,
+      "▁उपयोग": 91,
+      "कि": 92,
+      "के": 93,
+      "रण": 94,
+      "ाल": 95,
+      "20": 96,
+      "रा": 97,
+      "ेट": 98,
+      "ड़े": 99,
+      "दर्": 100,
+      "▁जि": 101,
+      "▁मा": 102,
+      "▁वि": 103,
+      "टी": 104,
+      "ती": 105,
+      "दा": 106,
+      "ूप": 107,
+      "▁य": 108,
+      "ेक्": 109,
+      "▁अन": 110,
+      "न्": 111,
+      "ूल": 112,
+      "▁थ": 113,
+      "▁रूप": 114,
+      "▁बड़े": 115,
+      "ाद": 116,
+      "▁इ": 117,
+      "▁जी": 118,
+      "▁या": 119,
+      "▁टोकन": 120,
+      "ंत": 121,
+      "पी": 122,
+      "ेटा": 123,
+      "▁पै": 124,
+      "▁हो": 125,
+      "िक्ष": 126,
+      "▁गया": 127,
+      "▁जाता": 128,
+      "▁प्रश": 129,
+      "ही": 130,
+      "्व": 131,
+      "मान": 132,
+      "र्म": 133,
+      "र्य": 134,
+      "▁[1": 135,
+      "▁सम": 136,
+      "▁साथ": 137,
+      "▁डेटा": 138,
+      "le": 139,
+      "ब्": 140,
+      "शन": 141,
+      "▁च": 142,
+      "ांस": 143,
+      "▁था": 144,
+      "▁नि": 145,
+      "▁ने": 146,
+      "▁अनु": 147,
+      "▁ट्र": 148,
+      "{\\": 149,
+      "ंग": 150,
+      "टर": 151,
+      "नी": 152,
+      "ला": 153,
+      "ल्": 154,
+      "▁1": 155,
+      "▁i": 156,
+      "र्क": 157,
+      "ूर्": 158,
+      "ैसे": 159,
+      "र्मर": 160,
+      "ांसफ": 161,
+      "▁उत्": 162,
+      "▁शब्": 163,
+      "▁करने": 164,
+      "ति": 165,
+      "नि": 166,
+      "मी": 167,
+      "ाव": 168,
+      "ोड": 169,
+      "201": 170,
+      "क्ष": 171,
+      "▁दे": 172,
+      "▁दो": 173,
+      "▁करता": 174,
+      "▁जीपी": 175,
+      "▁ट्रांसफ": 176,
+      "ty": 177,
+      "नक": 178,
+      "ीन": 179,
+      "▁पा": 180,
+      "▁बी": 181,
+      "▁यह": 182,
+      "▁सक": 183,
+      "▁क्ष": 184,
+      "▁जिस": 185,
+      "▁मूल": 186,
+      "▁कार्य": 187,
+      "▁प्रशिक्ष": 188,
+      "la": 189,
+      "ज़": 190,
+      "तर": 191,
+      "ष्": 192,
+      "कार": 193,
+      "धिक": 194,
+      "मता": 195,
+      "सेट": 196,
+      "▁आप": 197,
+      "▁गए": 198,
+      "▁वा": 199,
+      "▁हम": 200,
+      "दर्भ": 201,
+      "▁द्व": 202,
+      "▁वाल": 203,
+      "▁करना": 204,
+      "▁क्षमता": 205,
+      "▁संदर्भ": 206,
+      "▁डेटासेट": 207,
+      "di": 208,
+      "sp": 209,
+      "ys": 210,
+      "ंड": 211,
+      "आई": 212,
+      "ओं": 213,
+      "ढ़": 214,
+      "भी": 215,
+      "ले": 216,
+      "सं": 217,
+      "िल": 218,
+      "ौर": 219,
+      "▁ख": 220,
+      "▁फ": 221,
+      "त्र": 222,
+      "हीं": 223,
+      "िया": 224,
+      "्रे": 225,
+      "▁{\\": 226,
+      "▁इस": 227,
+      "▁कम": 228,
+      "▁तर": 229,
+      "▁रह": 230,
+      "disp": 231,
+      "lays": 232,
+      "tyle": 233,
+      "▁परि": 234,
+      "▁पाठ": 235,
+      "▁बना": 236,
+      "ार्मर": 237,
+      "▁अधिक": 238,
+      "▁शब्द": 239,
+      "▁प्रदर्": 240,
+      "displays": 241,
+      "displaystyle": 242,
+      "▁ट्रांसफार्मर": 243,
+      "वल": 244,
+      "हु": 245,
+      "ीक": 246,
+      "ुन": 247,
+      "क्त": 248,
+      "भाव": 249,
+      "स्त": 250,
+      "ारा": 251,
+      "ारी": 252,
+      "▁तक": 253,
+      "▁मश": 254,
+      "माने": 255,
+      "▁201": 256,
+      "▁एआई": 257,
+      "▁विश": 258,
+      "▁करते": 259,
+      "▁नहीं": 260,
+      "▁पूर्": 261,
+      "▁जीपीटी": 262,
+      "▁द्वारा": 263,
+      "▁पैमाने": 264,
+      "▁प्रदर्शन": 265,
+      "आर": 266,
+      "चर": 267,
+      "जी": 268,
+      "रे": 269,
+      "वि": 270,
+      "ाग": 271,
+      "▁)": 272,
+      "▁G": 273,
+      "इज़": 274,
+      "किट": 275,
+      "क्र": 276,
+      "दाह": 277,
+      "न्ह": 278,
+      "पाद": 279,
+      "ष्ट": 280,
+      "स्ट": 281,
+      "ोडर": 282,
+      "▁[3": 283,
+      "▁अप": 284,
+      "▁जब": 285,
+      "▁तो": 286,
+      "▁भी": 287,
+      "▁लग": 288,
+      "▁ही": 289,
+      "▁हु": 290,
+      "ंत्र": 291,
+      "इज़र": 292,
+      "▁आर्": 293,
+      "▁परत": 294,
+      "▁रहे": 295,
+      "▁विक": 296,
+      "▁समय": 297,
+      "दाहरण": 298,
+      "मान्य": 299,
+      "ेक्चर": 300,
+      "▁प्रत": 301,
+      "▁मशीन": 302,
+      "नाइज़र": 303,
+      "▁तंत्र": 304,
+      "▁आर्किट": 305,
+      "▁आर्किटेक्चर": 306,
+      "4]": 307,
+      "en": 308,
+      "eq": 309,
+      "og": 310,
+      "ok": 311,
+      "एम": 312,
+      "टव": 313,
+      "यू": 314,
+      "शल": 315,
+      "सी": 316,
+      "ाय": 317,
+      "ीय": 318,
+      "ुर": 319,
+      "ृत": 320,
+      "ेष": 321,
+      "▁ध": 322,
+      "ंतर": 323,
+      "ईआर": 324,
+      "करण": 325,
+      "खते": 326,
+      "बसे": 327,
+      "लता": 328,
+      "वाद": 329,
+      "िका": 330,
+      "ेंद": 331,
+      "▁[2": 332,
+      "▁": 333,
+      "क": 334,
+      "ा": 335,
+      "र": 336,
+      "े": 337,
+      "्": 338,
+      "ि": 339,
+      "न": 340,
+      "ं": 341,
+      "त": 342,
+      "म": 343,
+      "स": 344,
+      "प": 345,
+      "ह": 346,
+      "ल": 347,
+      "ी": 348,
+      "ो": 349,
+      "य": 350,
+      "ए": 351,
+      "ै": 352,
+      "ट": 353,
+      "व": 354,
+      "द": 355,
+      "ब": 356,
+      "ज": 357,
+      "ग": 358,
+      "ड": 359,
+      "श": 360,
+      ",": 361,
+      "।": 362,
+      "ू": 363,
+      "ष": 364,
+      "ु": 365,
+      "उ": 366,
+      "भ": 367,
+      "अ": 368,
+      "आ": 369,
+      "च": 370,
+      "1": 371,
+      "औ": 372,
+      "ण": 373,
+      "ॉ": 374,
+      "़": 375,
+      ")": 376,
+      "\"": 377,
+      "(": 378,
+      "0": 379,
+      "थ": 380,
+      "2": 381,
+      "ख": 382,
+      "ध": 383,
+      "[": 384,
+      "]": 385,
+      "e": 386,
+      "-": 387,
+      "ई": 388,
+      "इ": 389,
+      "फ": 390,
+      "l": 391,
+      "i": 392,
+      "s": 393,
+      "t": 394,
+      "{": 395,
+      "}": 396,
+      "o": 397,
+      "3": 398,
+      "y": 399,
+      "ौ": 400,
+      ".": 401,
+      "\\": 402,
+      "4": 403,
+      "ृ": 404,
+      "ठ": 405,
+      "a": 406,
+      "p": 407,
+      "r": 408,
+      "ओ": 409,
+      "छ": 410,
+      "9": 411,
+      "5": 412,
+      "7": 413,
+      "8": 414,
+      "k": 415,
+      "d": 416,
+      "ढ": 417,
+      "P": 418,
+      "n": 419,
+      "x": 420,
+      "ँ": 421,
+      ":": 422,
+      "G": 423,
+      "T": 424,
+      "我": 425,
+      "6": 426,
+      "M": 427,
+      "N": 428,
+      "g": 429,
+      "q": 430,
+      "झ": 431,
+      "S": 432,
+      "घ": 433,
+      "ञ": 434,
+      "'": 435,
+      "=": 436,
+      "?": 437,
+      "A": 438,
+      "V": 439,
+      "f": 440,
+      "型": 441,
+      "I": 442,
+      "L": 443,
+      "_": 444,
+      "c": 445,
+      "u": 446,
+      "ः": 447,
+      "ऑ": 448,
+      "$": 449,
+      "O": 450,
+      "m": 451,
+      "|": 452,
+      "⁡": 453,
+      "一": 454,
+      "个": 455,
+      "小": 456,
+      "是": 457,
+      "模": 458,
+      "的": 459,
+      "言": 460,
+      "语": 461,
+      "+": 462,
+      ";": 463,
+      ">": 464,
+      "C": 465,
+      "D": 466,
+      "Q": 467,
+      "^": 468,
+      "w": 469,
+      "ऊ": 470,
+      "ऐ": 471,
+      "∑": 472,
+      "−": 473,
+      "。": 474,
+      "不": 475,
+      "中": 476,
+      "主": 477,
+      "之": 478,
+      "了": 479,
+      "人": 480,
+      "会": 481,
+      "但": 482,
+      "其": 483,
+      "发": 484,
+      "员": 485,
+      "喜": 486,
+      "处": 487,
+      "复": 488,
+      "太": 489,
+      "常": 490,
+      "年": 491,
+      "开": 492,
+      "很": 493,
+      "微": 494,
+      "效": 495,
+      "有": 496,
+      "杂": 497,
+      "欢": 498,
+      "率": 499
+    },
+    "merges": [
+      [
+        "▁",
+        "क"
+      ],
+      [
+        "▁",
+        "प"
+      ],
+      [
+        "▁",
+        "ह"
+      ],
+      [
+        "▁",
+        "स"
+      ],
+      [
+        "▁",
+        "म"
+      ],
+      [
+        "▁ह",
+        "ै"
+      ],
+      [
+        "्",
+        "र"
+      ],
+      [
+        "र",
+        "्"
+      ],
+      [
+        "य",
+        "ा"
+      ],
+      [
+        "े",
+        "ं"
+      ],
+      [
+        "▁",
+        "के"
+      ],
+      [
+        "▁क",
+        "े"
+      ],
+      [
+        "▁",
+        "ज"
+      ],
+      [
+        "▁",
+        "ब"
+      ],
+      [
+        "▁",
+        "ए"
+      ],
+      [
+        "त",
+        "ा"
+      ],
+      [
+        "▁प",
+        "्र"
+      ],
+      [
+        "ि",
+        "क"
+      ],
+      [
+        "▁म",
+        "ें"
+      ],
+      [
+        "▁",
+        "उ"
+      ],
+      [
+        "न",
+        "े"
+      ],
+      [
+        "▁",
+        "अ"
+      ],
+      [
+        "▁क",
+        "र"
+      ],
+      [
+        "▁क",
+        "ा"
+      ],
+      [
+        "ो",
+        "ं"
+      ],
+      [
+        "न",
+        "ा"
+      ],
+      [
+        "▁",
+        "व"
+      ],
+      [
+        "▁",
+        "ल"
+      ],
+      [
+        "▁प",
+        "र"
+      ],
+      [
+        "▁",
+        "कि"
+      ],
+      [
+        "▁क",
+        "ि"
+      ],
+      [
+        "▁है",
+        "ं"
+      ],
+      [
+        "▁",
+        "औ"
+      ],
+      [
+        "▁औ",
+        "र"
+      ],
+      [
+        "ल",
+        "ए"
+      ],
+      [
+        "ि",
+        "त"
+      ],
+      [
+        "▁",
+        "द"
+      ],
+      [
+        "▁क",
+        "ो"
+      ],
+      [
+        "ा",
+        "र"
+      ],
+      [
+        "क",
+        "्"
+      ],
+      [
+        "▁",
+        "त"
+      ],
+      [
+        "▁ए",
+        "क"
+      ],
+      [
+        "▁",
+        "आ"
+      ],
+      [
+        "▁",
+        "ग"
+      ],
+      [
+        "▁क",
+        "ी"
+      ],
+      [
+        "ॉ",
+        "ड"
+      ],
+      [
+        "▁",
+        "("
+      ],
+      [
+        "▁म",
+        "ॉड"
+      ],
+      [
+        "स",
+        "े"
+      ],
+      [
+        "्",
+        "य"
+      ],
+      [
+        "▁",
+        "["
+      ],
+      [
+        "▁",
+        "ट"
+      ],
+      [
+        "▁",
+        "सं"
+      ],
+      [
+        "▁स",
+        "ं"
+      ],
+      [
+        "▁",
+        "से"
+      ],
+      [
+        "▁स",
+        "े"
+      ],
+      [
+        "▁मॉड",
+        "ल"
+      ],
+      [
+        "म",
+        "ा"
+      ],
+      [
+        "▁क",
+        "िया"
+      ],
+      [
+        "▁कि",
+        "या"
+      ],
+      [
+        "▁",
+        "भ"
+      ],
+      [
+        "▁",
+        "र"
+      ],
+      [
+        "त",
+        "े"
+      ],
+      [
+        "▁",
+        "न"
+      ],
+      [
+        "▁",
+        "श"
+      ],
+      [
+        "क",
+        "न"
+      ],
+      [
+        "ा",
+        "ं"
+      ],
+      [
+        "▁ल",
+        "ि"
+      ],
+      [
+        "स",
+        "्"
+      ],
+      [
+        "ड",
+        "़"
+      ],
+      [
+        "▁ज",
+        "ा"
+      ],
+      [
+        "्",
+        "ष"
+      ],
+      [
+        "▁लि",
+        "ए"
+      ],
+      [
+        "ो",
+        "ग"
+      ],
+      [
+        "ल",
+        "एम"
+      ],
+      [
+        "लए",
+        "म"
+      ],
+      [
+        "्",
+        "या"
+      ],
+      [
+        "्य",
+        "ा"
+      ],
+      [
+        "▁",
+        "\""
+      ],
+      [
+        "ा",
+        "ष"
+      ],
+      [
+        "▁उ",
+        "प"
+      ],
+      [
+        "▁ए",
+        "लए"
+      ],
+      [
+        "▁एलए",
+        "लएम"
+      ],
+      [
+        "त",
+        "्"
+      ],
+      [
+        "▁",
+        "ड"
+      ],
+      [
+        "य",
+        "ोग"
+      ],
+      [
+        "ाष",
+        "ा"
+      ],
+      [
+        "ो",
+        "कन"
+      ],
+      [
+        "▁ज",
+        "ो"
+      ],
+      [
+        "▁स",
+        "ा"
+      ],
+      [
+        "▁भ",
+        "ाषा"
+      ],
+      [
+        "▁उप",
+        "योग"
+      ],
+      [
+        "क",
+        "ि"
+      ],
+      [
+        "क",
+        "े"
+      ],
+      [
+        "र",
+        "ण"
+      ],
+      [
+        "ा",
+        "ल"
+      ],
+      [
+        "2",
+        "0"
+      ],
+      [
+        "र",
+        "ा"
+      ],
+      [
+        "े",
+        "ट"
+      ],
+      [
+        "ड़",
+        "े"
+      ],
+      [
+        "द",
+        "र्"
+      ],
+      [
+        "▁ज",
+        "ि"
+      ],
+      [
+        "▁",
+        "मा"
+      ],
+      [
+        "▁म",
+        "ा"
+      ],
+      [
+        "▁",
+        "वि"
+      ],
+      [
+        "▁व",
+        "ि"
+      ],
+      [
+        "ट",
+        "ी"
+      ],
+      [
+        "त",
+        "ी"
+      ],
+      [
+        "द",
+        "ा"
+      ],
+      [
+        "ू",
+        "प"
+      ],
+      [
+        "▁",
+        "य"
+      ],
+      [
+        "े",
+        "क्"
+      ],
+      [
+        "▁अ",
+        "न"
+      ],
+      [
+        "न",
+        "्"
+      ],
+      [
+        "ू",
+        "ल"
+      ],
+      [
+        "▁",
+        "थ"
+      ],
+      [
+        "▁र",
+        "ूप"
+      ],
+      [
+        "▁ब",
+        "ड़े"
+      ],
+      [
+        "ा",
+        "द"
+      ],
+      [
+        "▁",
+        "इ"
+      ],
+      [
+        "▁",
+        "जी"
+      ],
+      [
+        "▁ज",
+        "ी"
+      ],
+      [
+        "▁",
+        "या"
+      ],
+      [
+        "▁य",
+        "ा"
+      ],
+      [
+        "▁ट",
+        "ोकन"
+      ],
+      [
+        "ं",
+        "त"
+      ],
+      [
+        "प",
+        "ी"
+      ],
+      [
+        "ेट",
+        "ा"
+      ],
+      [
+        "▁प",
+        "ै"
+      ],
+      [
+        "▁ह",
+        "ो"
+      ],
+      [
+        "ि",
+        "क्ष"
+      ],
+      [
+        "िक",
+        "्ष"
+      ],
+      [
+        "▁ग",
+        "या"
+      ],
+      [
+        "▁जा",
+        "ता"
+      ],
+      [
+        "▁प्र",
+        "श"
+      ],
+      [
+        "ह",
+        "ी"
+      ],
+      [
+        "्",
+        "व"
+      ],
+      [
+        "मा",
+        "न"
+      ],
+      [
+        "र्",
+        "म"
+      ],
+      [
+        "र",
+        "्य"
+      ],
+      [
+        "र्",
+        "य"
+      ],
+      [
+        "▁[",
+        "1"
+      ],
+      [
+        "▁स",
+        "म"
+      ],
+      [
+        "▁सा",
+        "थ"
+      ],
+      [
+        "▁ड",
+        "ेटा"
+      ],
+      [
+        "l",
+        "e"
+      ],
+      [
+        "ब",
+        "्"
+      ],
+      [
+        "श",
+        "न"
+      ],
+      [
+        "▁",
+        "च"
+      ],
+      [
+        "ां",
+        "स"
+      ],
+      [
+        "▁थ",
+        "ा"
+      ],
+      [
+        "▁",
+        "नि"
+      ],
+      [
+        "▁न",
+        "ि"
+      ],
+      [
+        "▁",
+        "ने"
+      ],
+      [
+        "▁न",
+        "े"
+      ],
+      [
+        "▁अन",
+        "ु"
+      ],
+      [
+        "▁ट",
+        "्र"
+      ],
+      [
+        "{",
+        "\\"
+      ],
+      [
+        "ं",
+        "ग"
+      ],
+      [
+        "ट",
+        "र"
+      ],
+      [
+        "न",
+        "ी"
+      ],
+      [
+        "ल",
+        "ा"
+      ],
+      [
+        "ल",
+        "्"
+      ],
+      [
+        "▁",
+        "1"
+      ],
+      [
+        "▁",
+        "i"
+      ],
+      [
+        "र्",
+        "क"
+      ],
+      [
+        "ू",
+        "र्"
+      ],
+      [
+        "ै",
+        "से"
+      ],
+      [
+        "र्म",
+        "र"
+      ],
+      [
+        "ांस",
+        "फ"
+      ],
+      [
+        "▁उ",
+        "त्"
+      ],
+      [
+        "▁श",
+        "ब्"
+      ],
+      [
+        "▁कर",
+        "ने"
+      ],
+      [
+        "त",
+        "ि"
+      ],
+      [
+        "न",
+        "ि"
+      ],
+      [
+        "म",
+        "ी"
+      ],
+      [
+        "ा",
+        "व"
+      ],
+      [
+        "ो",
+        "ड"
+      ],
+      [
+        "20",
+        "1"
+      ],
+      [
+        "क",
+        "्ष"
+      ],
+      [
+        "क्",
+        "ष"
+      ],
+      [
+        "▁द",
+        "े"
+      ],
+      [
+        "▁द",
+        "ो"
+      ],
+      [
+        "▁कर",
+        "ता"
+      ],
+      [
+        "▁जी",
+        "पी"
+      ],
+      [
+        "▁ट्र",
+        "ांसफ"
+      ],
+      [
+        "t",
+        "y"
+      ],
+      [
+        "न",
+        "क"
+      ],
+      [
+        "ी",
+        "न"
+      ],
+      [
+        "▁प",
+        "ा"
+      ],
+      [
+        "▁ब",
+        "ी"
+      ],
+      [
+        "▁य",
+        "ह"
+      ],
+      [
+        "▁स",
+        "क"
+      ],
+      [
+        "▁",
+        "क्ष"
+      ],
+      [
+        "▁क",
+        "्ष"
+      ],
+      [
+        "▁जि",
+        "स"
+      ],
+      [
+        "▁म",
+        "ूल"
+      ],
+      [
+        "▁का",
+        "र्य"
+      ],
+      [
+        "▁प्रश",
+        "िक्ष"
+      ],
+      [
+        "l",
+        "a"
+      ],
+      [
+        "ज",
+        "़"
+      ],
+      [
+        "त",
+        "र"
+      ],
+      [
+        "ष",
+        "्"
+      ],
+      [
+        "क",
+        "ार"
+      ],
+      [
+        "ध",
+        "िक"
+      ],
+      [
+        "म",
+        "ता"
+      ],
+      [
+        "स",
+        "ेट"
+      ],
+      [
+        "से",
+        "ट"
+      ],
+      [
+        "▁आ",
+        "प"
+      ],
+      [
+        "▁ग",
+        "ए"
+      ],
+      [
+        "▁व",
+        "ा"
+      ],
+      [
+        "▁ह",
+        "म"
+      ],
+      [
+        "दर्",
+        "भ"
+      ],
+      [
+        "▁द",
+        "्व"
+      ],
+      [
+        "▁व",
+        "ाल"
+      ],
+      [
+        "▁वा",
+        "ल"
+      ],
+      [
+        "▁कर",
+        "ना"
+      ],
+      [
+        "▁क्ष",
+        "मता"
+      ],
+      [
+        "▁सं",
+        "दर्भ"
+      ],
+      [
+        "▁डेटा",
+        "सेट"
+      ],
+      [
+        "d",
+        "i"
+      ],
+      [
+        "s",
+        "p"
+      ],
+      [
+        "y",
+        "s"
+      ],
+      [
+        "ं",
+        "ड"
+      ],
+      [
+        "आ",
+        "ई"
+      ],
+      [
+        "ओ",
+        "ं"
+      ],
+      [
+        "ढ",
+        "़"
+      ],
+      [
+        "भ",
+        "ी"
+      ],
+      [
+        "ल",
+        "े"
+      ],
+      [
+        "स",
+        "ं"
+      ],
+      [
+        "ि",
+        "ल"
+      ],
+      [
+        "ौ",
+        "र"
+      ],
+      [
+        "▁",
+        "ख"
+      ],
+      [
+        "▁",
+        "फ"
+      ],
+      [
+        "त",
+        "्र"
+      ],
+      [
+        "त्",
+        "र"
+      ],
+      [
+        "ही",
+        "ं"
+      ],
+      [
+        "ि",
+        "या"
+      ],
+      [
+        "्",
+        "रे"
+      ],
+      [
+        "्र",
+        "े"
+      ],
+      [
+        "▁",
+        "{\\"
+      ],
+      [
+        "▁इ",
+        "स"
+      ],
+      [
+        "▁क",
+        "म"
+      ],
+      [
+        "▁",
+        "तर"
+      ],
+      [
+        "▁त",
+        "र"
+      ],
+      [
+        "▁र",
+        "ह"
+      ],
+      [
+        "di",
+        "sp"
+      ],
+      [
+        "la",
+        "ys"
+      ],
+      [
+        "ty",
+        "le"
+      ],
+      [
+        "▁पर",
+        "ि"
+      ],
+      [
+        "▁पा",
+        "ठ"
+      ],
+      [
+        "▁ब",
+        "ना"
+      ],
+      [
+        "ा",
+        "र्मर"
+      ],
+      [
+        "▁अ",
+        "धिक"
+      ],
+      [
+        "▁शब्",
+        "द"
+      ],
+      [
+        "▁प्र",
+        "दर्"
+      ],
+      [
+        "disp",
+        "lays"
+      ],
+      [
+        "displays",
+        "tyle"
+      ],
+      [
+        "▁ट्रांसफ",
+        "ार्मर"
+      ],
+      [
+        "व",
+        "ल"
+      ],
+      [
+        "ह",
+        "ु"
+      ],
+      [
+        "ी",
+        "क"
+      ],
+      [
+        "ु",
+        "न"
+      ],
+      [
+        "क्",
+        "त"
+      ],
+      [
+        "भ",
+        "ाव"
+      ],
+      [
+        "स्",
+        "त"
+      ],
+      [
+        "ा",
+        "रा"
+      ],
+      [
+        "ार",
+        "ा"
+      ],
+      [
+        "ार",
+        "ी"
+      ],
+      [
+        "▁त",
+        "क"
+      ],
+      [
+        "▁म",
+        "श"
+      ],
+      [
+        "मा",
+        "ने"
+      ],
+      [
+        "मान",
+        "े"
+      ],
+      [
+        "▁",
+        "201"
+      ],
+      [
+        "▁ए",
+        "आई"
+      ],
+      [
+        "▁वि",
+        "श"
+      ],
+      [
+        "▁कर",
+        "ते"
+      ],
+      [
+        "▁न",
+        "हीं"
+      ],
+      [
+        "▁प",
+        "ूर्"
+      ],
+      [
+        "▁जीपी",
+        "टी"
+      ],
+      [
+        "▁द्व",
+        "ारा"
+      ],
+      [
+        "▁पै",
+        "माने"
+      ],
+      [
+        "▁प्रदर्",
+        "शन"
+      ],
+      [
+        "आ",
+        "र"
+      ],
+      [
+        "च",
+        "र"
+      ],
+      [
+        "ज",
+        "ी"
+      ],
+      [
+        "र",
+        "े"
+      ],
+      [
+        "व",
+        "ि"
+      ],
+      [
+        "ा",
+        "ग"
+      ],
+      [
+        "▁",
+        ")"
+      ],
+      [
+        "▁",
+        "G"
+      ],
+      [
+        "इ",
+        "ज़"
+      ],
+      [
+        "कि",
+        "ट"
+      ],
+      [
+        "क",
+        "्र"
+      ],
+      [
+        "क्",
+        "र"
+      ],
+      [
+        "दा",
+        "ह"
+      ],
+      [
+        "न्",
+        "ह"
+      ],
+      [
+        "प",
+        "ाद"
+      ],
+      [
+        "ष्",
+        "ट"
+      ],
+      [
+        "स्",
+        "ट"
+      ],
+      [
+        "ोड",
+        "र"
+      ],
+      [
+        "▁[",
+        "3"
+      ],
+      [
+        "▁अ",
+        "प"
+      ],
+      [
+        "▁ज",
+        "ब"
+      ],
+      [
+        "▁त",
+        "ो"
+      ],
+      [
+        "▁",
+        "भी"
+      ],
+      [
+        "▁भ",
+        "ी"
+      ],
+      [
+        "▁ल",
+        "ग"
+      ],
+      [
+        "▁",
+        "ही"
+      ],
+      [
+        "▁ह",
+        "ी"
+      ],
+      [
+        "▁",
+        "हु"
+      ],
+      [
+        "▁ह",
+        "ु"
+      ],
+      [
+        "ं",
+        "त्र"
+      ],
+      [
+        "ंत",
+        "्र"
+      ],
+      [
+        "इज़",
+        "र"
+      ],
+      [
+        "▁आ",
+        "र्"
+      ],
+      [
+        "▁पर",
+        "त"
+      ],
+      [
+        "▁रह",
+        "े"
+      ],
+      [
+        "▁व",
+        "िक"
+      ],
+      [
+        "▁वि",
+        "क"
+      ],
+      [
+        "▁सम",
+        "य"
+      ],
+      [
+        "दाह",
+        "रण"
+      ],
+      [
+        "मान",
+        "्य"
+      ],
+      [
+        "ेक्",
+        "चर"
+      ],
+      [
+        "▁प्र",
+        "त"
+      ],
+      [
+        "▁मश",
+        "ीन"
+      ],
+      [
+        "ना",
+        "इज़र"
+      ],
+      [
+        "▁त",
+        "ंत्र"
+      ],
+      [
+        "▁आर्",
+        "किट"
+      ],
+      [
+        "▁आर्किट",
+        "ेक्चर"
+      ],
+      [
+        "4",
+        "]"
+      ],
+      [
+        "e",
+        "n"
+      ],
+      [
+        "e",
+        "q"
+      ],
+      [
+        "o",
+        "g"
+      ],
+      [
+        "o",
+        "k"
+      ],
+      [
+        "ए",
+        "म"
+      ],
+      [
+        "ट",
+        "व"
+      ],
+      [
+        "य",
+        "ू"
+      ],
+      [
+        "श",
+        "ल"
+      ],
+      [
+        "स",
+        "ी"
+      ],
+      [
+        "ा",
+        "य"
+      ],
+      [
+        "ी",
+        "य"
+      ],
+      [
+        "ु",
+        "र"
+      ],
+      [
+        "ृ",
+        "त"
+      ],
+      [
+        "े",
+        "ष"
+      ],
+      [
+        "▁",
+        "ध"
+      ],
+      [
+        "ं",
+        "तर"
+      ],
+      [
+        "ंत",
+        "र"
+      ],
+      [
+        "ई",
+        "आर"
+      ],
+      [
+        "क",
+        "रण"
+      ],
+      [
+        "ख",
+        "ते"
+      ],
+      [
+        "ब",
+        "से"
+      ],
+      [
+        "ल",
+        "ता"
+      ],
+      [
+        "व",
+        "ाद"
+      ],
+      [
+        "िक",
+        "ा"
+      ],
+      [
+        "ें",
+        "द"
+      ],
+      [
+        "▁[",
+        "2"
+      ]
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "extra_special_tokens": [
+    "<pad>",
+    "</s>",
+    "<s>",
+    "<mask>",
+    "प्रश्न:",
+    "उत्तर:"
+  ],
+  "is_local": true,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "vocab_file": "tokenizer-gemma/tokenizer.model"
+}