Update 3 files

- /c_tokenizer/main.c
- /c_tokenizer/tokenizer.h
- /c_tokenizer/Makefile

c_tokenizer/Makefile

@@ -0,0 +1,45 @@
+CC		:= gcc
+CFLAGS		:= -O3 -Ofast -ffast-math
+CLIBS		:= -fopenmp
+
+C_TOKENIZER	:= tokenizer.bin
+SRC		:= *.c tokenizer.o
+TARGET		:= a.out
+
+
+
+all:	$(TARGET)
+
+clean:	
+	$(RM) $(TARGET) *.o
+
+wipe:
+	make clean
+	$(RM) *.bin *.txt
+
+
+run:	$(TARGET)
+	./$< data.txt data.bin
+
+
+
+$(C_TOKENIZER):
+	awk 'BEGIN {for (i = 0; i <= 255; i++) printf("%c%c%c", i, 0, 0)}' > $@
+
+
+#elf64-littleaarch64
+
+
+tokenizer.o:	$(C_TOKENIZER)
+	objcopy --input-target binary \
+		--output-target elf64-x86-64 \
+		$< $@	
+
+		#--redefine-sym _binary_tokenizer_bin_start=_embedded_binary_tokenizer \
+		#$< $@
+
+
+
+
+$(TARGET):	$(SRC)
+	$(CC) $(CFLAGS) -o $@ $^ $(CLIBS)

c_tokenizer/main.c

@@ -0,0 +1,82 @@
+
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+
+
+
+
+
+#include "tokenizer.h"
+
+
+
+
+
+
+uint32_t get_file_size(const char *filename) {
+    FILE *file = fopen(filename, "rb");
+    if (!file) {
+        perror("fopen");
+        exit(EXIT_FAILURE);
+    }
+
+    if (fseek(file, 0, SEEK_END)!= 0) {
+        perror("fseek");
+        exit(EXIT_FAILURE);
+    }
+
+    uint32_t size = ftell(file);
+    if (size == -1) {
+        perror("ftell");
+        exit(EXIT_FAILURE);
+    }
+
+    fclose(file);
+    return size;
+}
+
+
+int main(int argc, char *argv[]) {
+	if(argc < 3) {
+		printf("Needs at least 2 arguments\n");
+		return 1;
+	}
+
+	char *fname[2] = {argv[1], argv[2]};
+	int fd[2] = {
+		open(fname[0], O_RDONLY), 
+		open(fname[1], O_CREAT | O_WRONLY, 0644)
+	};
+	
+	printf("Vocab size is %d\n", tokenizer.get_vocab_size());
+	printf("File names: %s, %s\n", fname[0], fname[1]);
+	printf("Got file descriptor: %d, %d\n", fd[0], fd[1]);
+
+	uint32_t size = get_file_size(fname[0]);
+	uint8_t *data = (uint8_t *) malloc(size * sizeof(uint8_t));
+	uint32_t n = read(fd[0], data, size * sizeof(uint8_t));
+
+	printf("Expected file size of %d bytes\n", size);
+	printf("Read %d bytes\n", n);
+	if (size != n) perror(0);
+
+	uint8_t *p = data;
+	uint32_t d = 0;
+	for (int i = 0; *p; ++i) {
+		uint16_t tok = tokenizer.encode(&tokenizer, &p);
+		write(fd[1], &tok, sizeof(uint16_t));
+		
+		if (i % 10000 == 0) {
+			printf("Current token %d | Index position %d | Remaining %d | Tokenization delta %ld\n", tok, i, d, d - ((data + n) - p));
+			d = (data + n) - p;
+		}
+
+	}
+
+	printf("\n");
+	return 0;
+}

c_tokenizer/tokenizer.h

@@ -0,0 +1,88 @@
+#pragma once
+
+
+#include <stdint.h>
+
+
+
+
+extern char _binary_tokenizer_bin_start[];
+extern char _binary_tokenizer_bin_end[];
+
+
+
+#define MAX_WORD_LEN 24
+
+typedef struct {
+	struct __attribute__((packed)) {
+		uint8_t  byte;
+		uint16_t prev;
+	} token[VOCAB_SIZE];
+} vocab_t;
+
+typedef struct Tokenizer Tokenizer;
+struct Tokenizer {
+	vocab_t *vocab;
+	
+	uint16_t  (*get_vocab_size) (void);
+
+	uint16_t  (*find)   (Tokenizer *, uint8_t, uint16_t);
+	uint16_t  (*encode) (Tokenizer *, uint8_t **);
+	uint8_t  *(*decode) (Tokenizer *, uint16_t);
+
+
+};
+
+
+static uint16_t TokenizerGetVocabSize() { return (_binary_tokenizer_bin_end - _binary_tokenizer_bin_start) / 3; }
+
+
+
+static uint16_t TokenizerFind(Tokenizer *tokenizer, uint8_t byte, uint16_t prev) {
+	
+	for (uint16_t i = prev; i < tokenizer->get_vocab_size(); ++i)
+		if (tokenizer->vocab->token[i].byte == byte && tokenizer->vocab->token[i].prev == prev)
+			return i;
+
+	return 0;
+}
+
+
+static uint16_t TokenizerEncode(Tokenizer *tokenizer, uint8_t **seed_text) {	
+
+	uint16_t prev = 0;
+	for (; **seed_text; ++*seed_text) {
+		uint16_t next = tokenizer->find(tokenizer, **seed_text, prev);
+		if (next == 0) break;	
+		prev = next;
+	}
+
+	return prev;
+}
+
+
+static uint8_t *TokenizerDecode(Tokenizer *tokenizer, uint16_t token) {
+
+	static uint8_t dest[MAX_WORD_LEN + 1];
+	dest[MAX_WORD_LEN] = '\0';
+
+	uint16_t prev = token;
+	uint16_t i = MAX_WORD_LEN - 1;
+
+	for (; prev && i > 0; prev = tokenizer->vocab->token[prev].prev, --i)
+		dest[i] = tokenizer->vocab->token[prev].byte;
+
+	return dest + i + 1;
+}
+
+
+Tokenizer tokenizer = {
+	.vocab 	 	= (vocab_t *) _binary_tokenizer_bin_start,
+
+	.get_vocab_size = TokenizerGetVocabSize,
+
+	.find   = TokenizerFind,
+	.encode = TokenizerEncode,
+	.decode = TokenizerDecode
+};
+