File size: 1,256 Bytes
969c687 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 | """Shared helpers for the root-cause diagnostic (docs/ROOT_CAUSE.md)."""
import sys
from pathlib import Path
import torch
ROOT = Path(__file__).resolve().parents[3]
sys.path.insert(0, str(ROOT)); sys.path.insert(0, str(ROOT / "tools"))
from chat_atom_native import load_model # noqa: E402,F401
from src.io.atomizer import Atomizer # noqa: E402,F401
CKPT = str(ROOT / "checkpoints/byte_tick_fieldsat/atom_native_step_4996000_d32_ms1M_lrd_b4828_fieldsat.pt")
PROMPTS = ["Bonjour, comment ça va ?", "Qui es-tu ?", "Il était une fois"]
OFFSETS = [("data/corpus_fr_medium.txt", o) for o in (50_000, 200_000, 400_000)] + [("data/corpus_fr_medium_s21.txt", 250_000)]
N = 512
def val_windows_bytes():
"""Exactly the 2048-transition val of docs/artifacts/field_leak_spectrum/eval_windows.py:
4 windows, text=raw[off:off+2048] decoded ignore, first 513 bytes (byte-tick packets)."""
out = []
for p, off in OFFSETS:
raw = (ROOT / p).read_bytes()
text = raw[off: off + 4 * N].decode("utf-8", errors="ignore")
b = text.encode("utf-8")[: N + 1]
out.append(b)
return out
def corpus_bytes():
return b"".join((ROOT / f).read_bytes() for f in ("data/corpus_fr_medium.txt", "data/corpus_fr_medium_s21.txt"))
|