File size: 1,601 Bytes
6183f29 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 |
#include <stdio.h>
#include <fcntl.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include "tokenizer.h"
uint32_t get_file_size(const char *filename) {
FILE *file = fopen(filename, "rb");
if (!file) {
perror("fopen");
exit(EXIT_FAILURE);
}
if (fseek(file, 0, SEEK_END)!= 0) {
perror("fseek");
exit(EXIT_FAILURE);
}
uint32_t size = ftell(file);
if (size == -1) {
perror("ftell");
exit(EXIT_FAILURE);
}
fclose(file);
return size;
}
int main(int argc, char *argv[]) {
if(argc < 3) {
printf("Needs at least 2 arguments\n");
return 1;
}
char *fname[2] = {argv[1], argv[2]};
int fd[2] = {
open(fname[0], O_RDONLY),
open(fname[1], O_CREAT | O_WRONLY, 0644)
};
printf("Vocab size is %d\n", tokenizer.get_vocab_size());
printf("File names: %s, %s\n", fname[0], fname[1]);
printf("Got file descriptor: %d, %d\n", fd[0], fd[1]);
uint32_t size = get_file_size(fname[0]);
uint8_t *data = (uint8_t *) malloc(size * sizeof(uint8_t));
uint32_t n = read(fd[0], data, size * sizeof(uint8_t));
printf("Expected file size of %d bytes\n", size);
printf("Read %d bytes\n", n);
if (size != n) perror(0);
uint8_t *p = data;
uint32_t d = 0;
for (int i = 0; *p; ++i) {
uint16_t tok = tokenizer.encode(&tokenizer, &p);
write(fd[1], &tok, sizeof(uint16_t));
if (i % 10000 == 0) {
printf("Current token %d | Index position %d | Remaining %d | Tokenization delta %ld\n", tok, i, d, d - ((data + n) - p));
d = (data + n) - p;
}
}
printf("\n");
return 0;
}
|