Update file tokenizer.h
Browse files- c_tokenizer/tokenizer.h +1 -1
c_tokenizer/tokenizer.h
CHANGED
|
@@ -43,7 +43,7 @@ static uint16_t TokenizerGetVocabSize() { return (_binary_tokenizer_bin_end - _b
|
|
| 43 |
static uint16_t TokenizerFind(Tokenizer *tokenizer, uint8_t byte, uint16_t prev) {
|
| 44 |
|
| 45 |
for (int i = 0; i < tokenizer->get_vocab_size(); ++i)
|
| 46 |
-
|
| 47 |
|
| 48 |
exit(0);
|
| 49 |
|
|
|
|
| 43 |
static uint16_t TokenizerFind(Tokenizer *tokenizer, uint8_t byte, uint16_t prev) {
|
| 44 |
|
| 45 |
for (int i = 0; i < tokenizer->get_vocab_size(); ++i)
|
| 46 |
+
printf("token %d: (%c, %d)\n", i, tokenizer->vocab[i].byte, tokenizer->vocab[i].prev);
|
| 47 |
|
| 48 |
exit(0);
|
| 49 |
|