flpelerin commited on
Commit
40bc463
·
1 Parent(s): 172f6d4

Update file tokenizer.h

Browse files
Files changed (1) hide show
  1. c_tokenizer/tokenizer.h +1 -1
c_tokenizer/tokenizer.h CHANGED
@@ -43,7 +43,7 @@ static uint16_t TokenizerGetVocabSize() { return (_binary_tokenizer_bin_end - _b
43
  static uint16_t TokenizerFind(Tokenizer *tokenizer, uint8_t byte, uint16_t prev) {
44
 
45
  for (int i = 0; i < tokenizer->get_vocab_size(); ++i)
46
- print("token %d: (%c, %d)\n", i, tokenizer->vocab[i].byte, tokenizer->vocab[i].prev);
47
 
48
  exit(0);
49
 
 
43
  static uint16_t TokenizerFind(Tokenizer *tokenizer, uint8_t byte, uint16_t prev) {
44
 
45
  for (int i = 0; i < tokenizer->get_vocab_size(); ++i)
46
+ printf("token %d: (%c, %d)\n", i, tokenizer->vocab[i].byte, tokenizer->vocab[i].prev);
47
 
48
  exit(0);
49