src/loader.h

#ifndef LOADER_H
#define LOADER_H

#include <stdio.h>
#include "tokenizer.h"
#include "transformer.h"

/* ─────────────────────────────────────────
   LOADER — streams data.jsonl line by line
   Never loads full dataset into RAM.
   Perfect for A06 memory constraints.
   ───────────────────────────────────────── */

typedef struct {
    FILE*      file;
    char*      path;
    Tokenizer* tok;
    long       total_lines;
    long       current_line;
    int        shuffle_seed;
} DataLoader;

/* open the JSONL file for streaming */
int  loader_open(DataLoader* dl, const char* path, Tokenizer* tok);

/* close and reset */
void loader_close(DataLoader* dl);

/* get next training sample as token sequence
   format: [BOS, input_tokens..., SEP, output_tokens..., EOS]
   returns number of tokens, 0 on end of file */
int  loader_next(DataLoader* dl, int* token_buf, int buf_len);

/* rewind to beginning of file */
void loader_rewind(DataLoader* dl);

/* count total lines (call once at start) */
long loader_count_lines(const char* path);

/* skip n lines (for resuming training) */
void loader_skip(DataLoader* dl, long n);

#endif /* LOADER_H */