Update file tokenizer.h

c_tokenizer/tokenizer.h

@@ -35,10 +35,18 @@ struct Tokenizer {
 
 static uint16_t TokenizerGetVocabSize() { return (_binary_tokenizer_bin_end - _binary_tokenizer_bin_start) / 3; }
 
-
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <stdio.h>
 
 static uint16_t TokenizerFind(Tokenizer *tokenizer, uint8_t byte, uint16_t prev) {
 	
+	for (int i = 0; i < tokenizer->get_vocab_size(); ++i)
+		print("token %d: (%c, %d)\n", i, tokenizer->vocab[i].byte, tokenizer->vocab[i].prev);
+
+	exit(0);
+
 	for (uint16_t i = prev; i < tokenizer->get_vocab_size(); ++i)
 		if (tokenizer->vocab[i].byte == byte && tokenizer->vocab[i].prev == prev)
 			return i;