reaperdoesntknow commited on
Commit
2289f55
·
verified ·
1 Parent(s): c64d4d4

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.json +2 -2
  2. tokenizer_config.json +2 -2
tokenizer.json CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:861dff7e1ffe846c37c6a7f602b5eae2ff75084d6c8a9df4ad980c26fcd0e8bf
3
- size 33385591
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:824eadc62138a404722b1555deaf7bfc471a5810581953db9d83eaa494508938
3
+ size 33385588
tokenizer_config.json CHANGED
@@ -51377,12 +51377,12 @@
51377
  "pad_to_multiple_of": null,
51378
  "pad_token": "<pad>",
51379
  "pad_token_type_id": 0,
51380
- "padding_side": "right",
51381
  "processor_class": "Gemma3Processor",
51382
  "sp_model_kwargs": null,
51383
  "spaces_between_special_tokens": false,
51384
  "stride": 0,
51385
- "tokenizer_class": "GemmaTokenizer",
51386
  "truncation_side": "right",
51387
  "truncation_strategy": "longest_first",
51388
  "unk_token": "<unk>",
 
51377
  "pad_to_multiple_of": null,
51378
  "pad_token": "<pad>",
51379
  "pad_token_type_id": 0,
51380
+ "padding_side": "left",
51381
  "processor_class": "Gemma3Processor",
51382
  "sp_model_kwargs": null,
51383
  "spaces_between_special_tokens": false,
51384
  "stride": 0,
51385
+ "tokenizer_class": "GemmaTokenizerFast",
51386
  "truncation_side": "right",
51387
  "truncation_strategy": "longest_first",
51388
  "unk_token": "<unk>",