docs/tokenizer_bpe_pipeline.tex

\documentclass[tikz,border=8pt]{standalone}
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{tikz}
\usetikzlibrary{arrows.meta,backgrounds,positioning,shapes.geometric}

\definecolor{slayerRed}{RGB}{190,42,38}
\definecolor{ink}{RGB}{28,30,34}
\definecolor{muted}{RGB}{93,98,108}
\definecolor{panel}{RGB}{247,248,250}
\definecolor{line}{RGB}{190,196,205}

\tikzset{
  box/.style={
    draw=line,
    fill=panel,
    rounded corners=3pt,
    very thick,
    minimum width=3.2cm,
    minimum height=1.18cm,
    align=center,
    text=ink,
    font=\sffamily\small
  },
  note/.style={
    draw=slayerRed,
    fill=slayerRed!7,
    rounded corners=3pt,
    very thick,
    minimum width=11.4cm,
    minimum height=1.25cm,
    align=center,
    text=ink,
    font=\sffamily\small
  },
  arrow/.style={-{Latex[length=3mm]}, very thick, draw=ink},
  small/.style={font=\sffamily\footnotesize, text=muted, align=center}
}

\begin{document}
\begin{tikzpicture}[node distance=1.05cm and 0.95cm]
  \node[box] (raw) {Polski korpus\\UTF-8 text};
  \node[box, right=of raw] (bytes) {Byte-level\\pretokenizacja};
  \node[box, right=of bytes] (pairs) {Statystyczne\\zliczanie par};
  \node[box, right=of pairs] (merges) {BPE merges\\najczestsze pary};
  \node[box, right=of merges] (vocab) {Vocab 32k\\token ids uint16};

  \draw[arrow] (raw) -- (bytes);
  \draw[arrow] (bytes) -- (pairs);
  \draw[arrow] (pairs) -- (merges);
  \draw[arrow] (merges) -- (vocab);

  \node[note, below=1.05cm of pairs, xshift=2.05cm] (stat) {
    To nie jest tokenizer morfologiczny: brak reguł fleksyjnych, lematyzacji i słownika morfemów\\
    To czysty tokenizer statystyczny: segmenty wynikają z częstości par w korpusie
  };

  \draw[arrow, slayerRed] (pairs.south) -- ([xshift=-2.2cm]stat.north);
  \draw[arrow, slayerRed] (merges.south) -- (stat.north);

  \node[small, above=0.35cm of bytes] {diakrytyki sa zachowane jako bajty UTF-8};
  \node[small, above=0.35cm of vocab] {model/ckpt.pt wymaga\\tokenizers/polish\_bpe\_32k.json};

  \node[font=\sffamily\bfseries\large, text=ink, above=1.0cm of pairs] {
    Jak powstal tokenizer Slayer GPT: byte-level BPE, nie morfologia
  };

  \begin{scope}[on background layer]
    \fill[white] ([xshift=-0.35cm,yshift=-0.35cm]current bounding box.south west)
      rectangle ([xshift=0.35cm,yshift=0.35cm]current bounding box.north east);
  \end{scope}
\end{tikzpicture}
\end{document}