Premier modèle d'architecture gemma from scratch

README.md

@@ -39,7 +39,7 @@ The following hyperparameters were used during training:
 - total_train_batch_size: 32
 - optimizer: Use OptimizerNames.ADAMW_TORCH_FUSED with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: linear
-- num_epochs: 500
+- num_epochs: 3
 - mixed_precision_training: Native AMP
 
 ### Training results

tokenizer.json

@@ -150,16 +150,53 @@
       "<mask>": 4,
       "Ever:": 5,
       "Small:": 6,
-      "e": 7,
-      "C": 8,
-      "r": 9,
-      "▁": 10,
-      "a": 11,
-      "c": 12,
-      "i": 13,
-      "l": 14,
-      "m": 15,
-      "t": 16
+      "▁": 7,
+      "a": 8,
+      "i": 9,
+      "e": 10,
+      "n": 11,
+      "l": 12,
+      "o": 13,
+      "r": 14,
+      "y": 15,
+      "m": 16,
+      "t": 17,
+      "u": 18,
+      "c": 19,
+      "s": 20,
+      "p": 21,
+      "k": 22,
+      "d": 23,
+      "j": 24,
+      "z": 25,
+      "g": 26,
+      "b": 27,
+      "h": 28,
+      "v": 29,
+      "A": 30,
+      "f": 31,
+      "q": 32,
+      "x": 33,
+      "I": 34,
+      "Y": 35,
+      "E": 36,
+      "D": 37,
+      "L": 38,
+      "M": 39,
+      "N": 40,
+      "O": 41,
+      "w": 42,
+      "F": 43,
+      "P": 44,
+      "U": 45,
+      "W": 46,
+      "X": 47,
+      "B": 48,
+      "G": 49,
+      "J": 50,
+      "Q": 51,
+      "S": 52,
+      "T": 53
     },
     "merges": []
   }