ATOM (atom-ai): code snapshot (100fae0 + batched-train), README with measured numbers, curated checkpoints (part 3)
969c687 verified | """Shared helpers for the root-cause diagnostic (docs/ROOT_CAUSE.md).""" | |
| import sys | |
| from pathlib import Path | |
| import torch | |
| ROOT = Path(__file__).resolve().parents[3] | |
| sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "tools")) | |
| from chat_atom_native import load_model # noqa: E402,F401 | |
| from src.io.atomizer import Atomizer # noqa: E402,F401 | |
| CKPT = str(ROOT / "checkpoints/byte_tick_fieldsat/atom_native_step_4996000_d32_ms1M_lrd_b4828_fieldsat.pt") | |
| PROMPTS = ["Bonjour, comment ça va ?", "Qui es-tu ?", "Il était une fois"] | |
| OFFSETS = [("data/corpus_fr_medium.txt", o) for o in (50_000, 200_000, 400_000)] + [("data/corpus_fr_medium_s21.txt", 250_000)] | |
| N = 512 | |
| def val_windows_bytes(): | |
| """Exactly the 2048-transition val of docs/artifacts/field_leak_spectrum/eval_windows.py: | |
| 4 windows, text=raw[off:off+2048] decoded ignore, first 513 bytes (byte-tick packets).""" | |
| out = [] | |
| for p, off in OFFSETS: | |
| raw = (ROOT / p).read_bytes() | |
| text = raw[off: off + 4 * N].decode("utf-8", errors="ignore") | |
| b = text.encode("utf-8")[: N + 1] | |
| out.append(b) | |
| return out | |
| def corpus_bytes(): | |
| return b"".join((ROOT / f).read_bytes() for f in ("data/corpus_fr_medium.txt", "data/corpus_fr_medium_s21.txt")) | |