Tokenizer CPU

README.md

@@ -10,8 +10,10 @@ tags:
 
 # Tokenizer CPU
 
-A stock Hugging Face tokenizer that runs Brainfuck. It is a plain `tokenizer.json`, with no custom
-code and no model (no `trust_remote_code`). Every clock tick is one encode and one decode:
+A stock Hugging Face tokenizer that executes programs. Its merge rules select a machine transition;
+decoding writes the next state. Repeating encode and decode runs the computation, without a neural
+network. It is a plain `tokenizer.json`, with no custom code (no `trust_remote_code`). Every clock tick
+is one encode and one decode:
 
 ```python
 from huggingface_hub import hf_hub_download
@@ -44,12 +46,18 @@ changing.
 
 The full explanation, the compiler and the tests are on GitHub: https://github.com/fMindstormer/tokcpu
 
+Tag `v1.0` of this repo holds the same tokenizer files as the
+[v1.0 release](https://github.com/fMindstormer/tokcpu/releases/tag/v1.0) on GitHub, which adds a
+minimal runner, the expected results, checksums, and a reproduction of a tokenizer disagreement between
+Python and JavaScript.
+
 ## What's in this repo
 
 | Folder | Machine | Merge rules |
 |---|---|---:|
 | (root) | universal Brainfuck interpreter: runs any program written on the tape | 17,210 |
 | `hello`, `alphabet`, `grid`, `jumps` | one Brainfuck program compiled into each tokenizer | 24,173 – 38,265 |
+| `increment` | adds one to a binary number: `start.txt` holds 1011, which becomes 1100 after 8 ticks | 16 |
 | `4-state-busy-beaver` | the 4-state busy beaver: halts after 107 ticks with 13 ones | 24 |
 | `binary-counter` | counts in binary forever | 7 |
 | `sort` | a block cellular automaton: every two-letter block of the tape is rewritten in one tick, sorting it | 704 |

increment/start.txt

@@ -0,0 +1 @@
+⊢ⓐ1011⊣

increment/tokenizer.json

@@ -0,0 +1 @@
+{"version":"1.0","truncation":null,"padding":null,"added_tokens":[{"id":27,"content":"0ⓐ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":28,"content":"1Ⓒ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":29,"content":"1ⓐ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":30,"content":"Ⓑ0","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":31,"content":"Ⓑ_","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":32,"content":"1Ⓒ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":33,"content":"⊢Ⓑ_","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":34,"content":"⊢1Ⓒ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":35,"content":"0ⓐ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":36,"content":"1ⓐ","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":37,"content":"Ⓑ_","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":38,"content":"Ⓑ_⊣","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":39,"content":"ⓒ1","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":40,"content":"Ⓑ0","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":41,"content":"ⓒ1","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false},{"id":42,"content":"ⓒ1⊣","single_word":false,"lstrip":false,"rstrip":false,"normalized":false,"special":false}],"normalizer":null,"pre_tokenizer":null,"post_processor":null,"decoder":{"type":"Sequence","decoders":[{"type":"Replace","pattern":{"Regex":"[-]"},"content":""},{"type":"Fuse"}]},"model":{"type":"BPE","dropout":null,"unk_token":null,"continuing_subword_prefix":null,"end_of_word_suffix":null,"fuse_unk":false,"byte_fallback":false,"ignore_merges":false,"vocab":{"0":0,"1":1,"_":2,"⊢":3,"⊣":4,"Ⓐ":5,"Ⓑ":6,"Ⓒ":7,"ⓐ":8,"ⓑ":9,"ⓒ":10,"0Ⓐ":27,"0Ⓑ":28,"1Ⓐ":29,"1Ⓑ":30,"_Ⓐ":31,"_Ⓑ":32,"⊢Ⓐ":33,"⊢Ⓑ":34,"ⓐ0":35,"ⓐ1":36,"ⓐ_":37,"ⓐ⊣":38,"ⓑ0":39,"ⓑ1":40,"ⓑ_":41,"ⓑ⊣":42},"merges":[["0","Ⓐ"],["0","Ⓑ"],["1","Ⓐ"],["1","Ⓑ"],["_","Ⓐ"],["_","Ⓑ"],["⊢","Ⓐ"],["⊢","Ⓑ"],["ⓐ","0"],["ⓐ","1"],["ⓐ","_"],["ⓐ","⊣"],["ⓑ","0"],["ⓑ","1"],["ⓑ","_"],["ⓑ","⊣"]]}}

increment/tokenizer_config.json

@@ -0,0 +1 @@
+{"tokenizer_class": "PreTrainedTokenizerFast", "clean_up_tokenization_spaces": false, "model_max_length": 1000000000}