src/tokenizer.h

#ifndef TOKENIZER_H
#define TOKENIZER_H

#include <stddef.h>

/* ─────────────────────────────────────────
   TOKENIZER
   Character-level with word boundaries.
   Vocab size: 2048 (fits A06 RAM easily)
   ───────────────────────────────────────── */

#define TOK_VOCAB_SIZE   4096
#define TOK_MAX_WORD_LEN 32
#define TOK_UNK          0
#define TOK_BOS          1    /* <bos> beginning of sequence */
#define TOK_EOS          2    /* <eos> end of sequence       */
#define TOK_PAD          3    /* <pad> padding               */
#define TOK_SEP          4    /* <sep> input/output separator */

typedef struct {
    char    token[TOK_MAX_WORD_LEN];  /* token string */
    int     freq;                      /* frequency during build */
} TokenEntry;

typedef struct {
    TokenEntry  vocab[TOK_VOCAB_SIZE];
    int         vocab_size;
    /* hash map: token_string → id (simple open addressing) */
    int         hash_keys[TOK_VOCAB_SIZE * 4];
    int         hash_vals[TOK_VOCAB_SIZE * 4];
    int         hash_size;
} Tokenizer;

/* build vocab from a JSONL file (reads input+output fields) */
int  tokenizer_build(Tokenizer* tok, const char* jsonl_path, int max_lines);

/* save/load vocab */
int  tokenizer_save(const Tokenizer* tok, const char* path);
int  tokenizer_load(Tokenizer* tok, const char* path);

/* encode text → token ids. returns number of tokens written.
   caller must provide buf of size buf_len */
int  tokenizer_encode(const Tokenizer* tok, const char* text,
                      int* buf, int buf_len);

/* decode token ids → text string */
void tokenizer_decode(const Tokenizer* tok, const int* ids, int n,
                      char* out, int out_len);

/* single token lookup */
int  tokenizer_token_to_id(const Tokenizer* tok, const char* token);
const char* tokenizer_id_to_token(const Tokenizer* tok, int id);

#endif /* TOKENIZER_H */