Update file tokenizer.h

c_tokenizer/tokenizer.h

@@ -40,7 +40,7 @@ static uint16_t TokenizerGetVocabSize() { return (_binary_tokenizer_bin_end - _b
 static uint16_t TokenizerFind(Tokenizer *tokenizer, uint8_t byte, uint16_t prev) {
 	
 	for (uint16_t i = prev; i < tokenizer->get_vocab_size(); ++i)
-		if (tokenizer->vocab->token[i].byte == byte && tokenizer->vocab->token[i].prev == prev)
+		if (tokenizer->vocab[i].byte == byte && tokenizer->vocab[i].prev == prev)
 			return i;
 
 	return 0;
@@ -68,8 +68,8 @@ static uint8_t *TokenizerDecode(Tokenizer *tokenizer, uint16_t token) {
 	uint16_t prev = token;
 	uint16_t i = MAX_WORD_LEN - 1;
 
-	for (; prev && i > 0; prev = tokenizer->vocab->token[prev].prev, --i)
-		dest[i] = tokenizer->vocab->token[prev].byte;
+	for (; prev && i > 0; prev = tokenizer->vocab[prev].prev, --i)
+		dest[i] = tokenizer->vocab[prev].byte;
 
 	return dest + i + 1;
 }