philippyt's picture
MVP1: working prototype of transformer ablation playground. web app for composing tiny transformer architectures, running real inference, and seeing how each architectural switch affects cost and quality. Hash-stable arch_config shared between Python and TS; closed-form cost formulas parity-tested against actual sum(p.numel())
84702bc
Raw History Blame Contribute Delete
971 Bytes
from __future__ import annotations
from pathlib import Path
import torch
from ablation.tokenizer import CharTokenizer
def load_text(dataset_path: str) -> str:
path = Path(dataset_path)
if not path.is_absolute():
path = Path(__file__).resolve().parent.parent / path
return path.read_text(encoding="utf-8")
def make_split(text: str, tokenizer: CharTokenizer, val_frac: float = 0.1) -> tuple[torch.Tensor, torch.Tensor]:
ids = torch.tensor(tokenizer.encode(text), dtype=torch.long)
n_val = int(len(ids) * val_frac)
n_train = len(ids) - n_val
return ids[:n_train], ids[n_train:]
def get_batch(data: torch.Tensor, *, batch_size: int, ctx: int, device: torch.device | str) -> tuple[torch.Tensor, torch.Tensor]:
ix = torch.randint(low=0, high=len(data) - ctx - 1, size=(batch_size,))
x = torch.stack([data[i : i + ctx] for i in ix])
y = torch.stack([data[i + 1 : i + 1 + ctx] for i in ix])
return x.to(device), y.to(device)