hf-internal-testing
/

fuyu-tiny-tokenizer

hf-transformers-bot commited on Jun 30

Commit

102357e

verified ·

1 Parent(s): 724c0e8

Add tiny Fuyu tokenizer fixture for testing

Files changed (1) hide show

tokenizer.json CHANGED Viewed

@@ -13,7 +13,7 @@
       "special": true
     },
     {
-      "id": 503,
       "content": "|ENDOFTEXT|",
       "single_word": false,
       "lstrip": false,
@@ -85,7 +85,7 @@
       "|ENDOFTEXT|": {
         "id": "|ENDOFTEXT|",
         "ids": [
-          503
         ],
         "tokens": [
           "|ENDOFTEXT|"
@@ -2129,6 +2129,26 @@
         "|NEWLINE|",
         0.0
       ],
       [
         "▁",
         -1.457676649093628

       "special": true
     },
     {
+      "id": 508,
       "content": "|ENDOFTEXT|",
       "single_word": false,
       "lstrip": false,
       "|ENDOFTEXT|": {
         "id": "|ENDOFTEXT|",
         "ids": [
+          508
         ],
         "tokens": [
           "|ENDOFTEXT|"
         "|NEWLINE|",
         0.0
       ],
+      [
+        "<0x00>",
+        0.0
+      ],
+      [
+        "<0x01>",
+        0.0
+      ],
+      [
+        "<0x02>",
+        0.0
+      ],
+      [
+        "<0x03>",
+        0.0
+      ],
+      [
+        "<0x04>",
+        0.0
+      ],
       [
         "▁",
         -1.457676649093628