alter_ego / training /kernels /preprocess.py
geraldaton20's picture
initial upload: source code (no build artifacts)
f4f166e verified
Raw History Blame Contribute Delete
24 kB
"""Stage 3 preprocess kernel v2: streaming mix -> dedup -> perplexity filter ->
quality top-80% -> BPE-8192 tokenize -> packed numpy corpus -> Kaggle dataset
ae-corpus-v1 (private). Self-contained. Resumable via HF-mirrored manifest.
v2 fixes (2026-08-24, from first-run log):
- ultrachat split train_sft (train doesn't exist)
- synthcot source swapped to Open-Orca/SlimOrca (open-phi/synthetic-cot DNE)
- stack_py: HF token passed to load_dataset (gated repo); still requires the
account to have accepted bigcode terms once on the Hub - else auto-dropped
- storage: zarr-v3 removed Group.create_dataset -> replaced with dependency-
free append-only tokens.i16 + atomic arrays.npz
Outputs:
/kaggle/work/corpus/ tokens.i16, arrays.npz {lengths,scores,src},
tokenizer.json, corpus_meta.json
HF <ckpt_repo> preprocess/manifest.jsonl, preprocess/tokenizer.json,
telemetry/preprocess_stats.json, preprocess/DONE
Secret aliases required: HF_TOKEN, KAGGLE_USERNAME, KAGGLE_API_TOKEN.
"""
import hashlib
import json
import os
import subprocess
import sys
import time
import numpy as np
REPO = os.environ.get("HF_CKPT_REPO", "geraldaton20/student-ckpts-v1")
OUT = os.environ.get("AE_OUT", "/kaggle/work/corpus")
LOCAL = os.environ.get("AE_LOCAL", "0") == "1"
PUBLISH = os.environ.get("AE_PUBLISH", "" ) != "0"
HF_PREFIX = "preprocess"
TARGET_TOKENS = int(os.environ.get("TARGET_TOKENS", "850000000"))
CHUNK_DOCS = int(os.environ.get("CHUNK_DOCS", "20000"))
TOKENIZER_SAMPLE_DOCS = int(os.environ.get("TOKENIZER_SAMPLE_DOCS", "60000"))
VOCAB = 8192
MIN_LEN, MAX_LEN = 50, 256
PPL_MIN, PPL_MAX = 5.0, 50.0
PROBE_MODEL = "gpt2"
QUALITY_KEEP_FRAC = 0.80
MAX_MINUTES = int(os.environ.get("MAX_MINUTES", "360"))
SEED = 42
SOURCES = [
{"name": "cosmopedia", "repo": "HuggingFaceTB/cosmopedia",
"config": "stanford", "ratio": 0.40, "split": "train"},
{"name": "stack_py", "repo": "bigcode/the-stack-v2",
"config": None, "ratio": 0.30, "field": "content", "split": "train",
"gated": True,
# non-gated fallback so the pipeline never blocks on token scopes:
"alt": {"repo": "code_search_net", "config": "python",
"ratio": 0.30, "field": "func_code_string", "split": "train"}},
{"name": "ultrachat", "repo": "openbmb/UltraChat",
"config": None, "ratio": 0.20, "field": "data", "split": "train"},
{"name": "synthcot", "repo": "casperhansen/gsm8k_synthetic_cot",
"config": None, "ratio": 0.10, "split": "train",
"join_fields": ["question", "cot", "answer"]},
]
# SPEC CORRECTIONS 2026-08-24: open-phi/synthetic-cot does not exist ->
# casperhansen/gsm8k_synthetic_cot (user-selected); ultrachat moved to the
# original openbmb/UltraChat (single 'train' split, 'data' turn list).
def install_deps():
missing = []
for m in ("datasets", "transformers", "tokenizers", "huggingface_hub"):
try:
__import__(m)
except Exception:
missing.append(m)
if missing:
subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True)
print("[bootstrap] deps ok", flush=True)
def hf_token():
tok = os.environ.get("HF_TOKEN")
if tok:
return tok
try:
from kaggle_secrets import UserSecretsClient
return UserSecretsClient().get_secret("HF_TOKEN")
except Exception:
return None
class HF:
def __init__(self):
self.token = hf_token()
from huggingface_hub import HfApi
self.api = HfApi(token=self.token) if self.token else None
def push_file(self, local, path_in_repo):
if not self.api:
return
last = None
for i in range(4):
try:
self.api.upload_file(path_or_fileobj=local, path_in_repo=path_in_repo,
repo_id=REPO, repo_type="model")
return
except Exception as e:
last = e
time.sleep(2 * (i + 1))
print(f"[hf] push {path_in_repo} failed: {last}", flush=True)
def download(self, path_in_repo, local_dir):
if not self.api:
return None
try:
return self.api.hf_hub_download(repo_id=REPO, filename=path_in_repo,
repo_type="model", local_dir=local_dir)
except Exception:
return None
def sha_bytes(b):
return hashlib.sha256(b).hexdigest()[:16]
def extract_text(ex):
src = ex.get("__src", "")
spec = next((s for s in SOURCES if s["name"] == src), {})
if spec.get("join_fields"):
parts = []
for f in spec["join_fields"]:
v = ex.get(f)
if isinstance(v, list):
parts.extend(x for x in v if isinstance(x, str))
elif isinstance(v, str):
parts.append(v)
if parts:
return src, "\n".join(parts)
f = spec.get("field")
if f and f in ex:
v = ex[f]
if isinstance(v, str):
return src, v
if isinstance(v, list):
out = []
for m in v:
if isinstance(m, str):
out.append(m)
elif isinstance(m, dict):
for k in ("content", "text", "value"):
if isinstance(m.get(k), str):
out.append(m[k])
break
return src, "\n".join(out)
for k in ("text", "content", "question", "instruction", "prompt"):
v = ex.get(k)
if isinstance(v, str) and v.strip():
return src, v
return src, ""
def build_stream():
"""Load every source (with optional non-gated fallbacks) and interleave."""
from datasets import interleave_datasets, load_dataset
def try_one(spec):
kw = dict(path=spec["repo"], split=spec["split"], streaming=True,
token=hf_token())
if spec.get("config"):
kw["name"] = spec["config"]
return load_dataset(**kw)
sets, probs, dropped = [], [], []
for spec in SOURCES:
candidates = [spec] + ([spec["alt"]] if spec.get("alt") else [])
loaded = None
for cand in candidates:
try:
ds = try_one(cand)
loaded = (ds, cand)
tag = "" if cand is spec else " [fallback]"
print(f"[stream] {spec['name']}{tag} OK ({cand['ratio']:.0%})",
flush=True)
break
except Exception as e:
hint = ""
if cand.get("gated") and "gated" in str(e).lower():
hint = (" (gated: token must cover this repo - a CLASSIC "
"token carries account grants; fine-grained needs "
f"the repo explicitly selected)")
print(f"[stream] try {cand['repo']} FAILED ({e}){hint}", flush=True)
if loaded is None:
dropped.append(spec["name"])
continue
ds, cand = loaded
# merge the winning candidate's extraction hints onto the source spec
for k in ("field", "join_fields", "config"):
if k in cand:
spec[k] = cand[k]
name = spec["name"]
ds = ds.map(lambda ex, _n=name: {"__src": _n})
sets.append(ds)
probs.append(spec["ratio"])
if not sets:
raise SystemExit("no sources loaded")
tot = sum(probs)
probs = [p / tot for p in probs]
mixed = interleave_datasets(sets, probabilities=probs, seed=SEED)
return mixed, dropped
class ExactDedup:
def __init__(self, cap=20_000_000):
self.seen = set()
self.cap = cap
def check(self, text):
h = int.from_bytes(hashlib.blake2b(
" ".join(text.lower().split())[:4000].encode(),
digest_size=8).digest(), "little")
if h in self.seen:
return False
if len(self.seen) < self.cap:
self.seen.add(h)
return True
def near_dup_filter(items):
"""Per-chunk MinHash LSH near-dup removal (chunk-scoped, idempotent)."""
try:
from datasketch import MinHash, MinHashLSH
except Exception:
return items
lsh = MinHashLSH(threshold=0.85, num_perm=128)
out = []
for j, it in enumerate(items):
words = it["text"].split()
m = MinHash(num_perm=128)
n = 0
for s in range(0, max(1, len(words) - 4), 3):
m.update(" ".join(words[s:s + 5]).encode("utf8"))
n += 1
if n >= 200:
break
if n == 0:
out.append(it)
continue
dup = False
try:
for _key in lsh.query(m):
dup = True
break
except Exception:
pass
if not dup:
try:
lsh.insert(str(j), m)
except Exception:
pass
out.append(it)
return out
def quality_score(text):
score = 0.0
if any(kw in text for kw in ("def ", "class ", "import ", "function")):
score += 0.3
if any(marker in text for marker in ("Therefore,", "Step 1:", "First,", "Because")):
score += 0.2
toks = len(text.split())
if toks < 30:
score -= 0.3
elif toks > 200:
score += 0.1
uniq = len(set(text.split())) / max(1, len(text.split()))
return max(0.0, min(1.0, score + uniq * 0.2))
class PplProbe:
def __init__(self):
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
self.torch = torch
self.dev = "cuda" if torch.cuda.is_available() else "cpu"
self.tk = AutoTokenizer.from_pretrained(PROBE_MODEL)
if self.tk.pad_token_id is None:
self.tk.pad_token = self.tk.eos_token
dt = torch.float16 if self.dev == "cuda" else torch.float32
self.model = AutoModelForCausalLM.from_pretrained(
PROBE_MODEL, torch_dtype=dt).to(self.dev).eval()
def ppl_batch(self, texts, batch=32):
torch = self.torch
out = []
with torch.no_grad():
for s in range(0, len(texts), batch):
part = [t[:1500] for t in texts[s:s + batch]]
enc = self.tk(part, return_tensors="pt", padding=True,
truncation=True, max_length=256).to(self.dev)
logits = self.model(**enc).logits.float()
labels = enc["input_ids"][:, 1:]
mask = enc["attention_mask"][:, 1:].float()
nll = torch.nn.functional.cross_entropy(
logits[:, :-1].reshape(-1, logits.size(-1)),
labels.reshape(-1), reduction="none").view(labels.shape)
nll = (nll * mask).sum(1) / mask.sum(1).clamp(min=1)
out.extend(torch.exp(nll).tolist())
return out
class Corpus:
"""Dependency-free corpus store: tokens.i16 (append-only, int16 LE) +
arrays.npz {lengths i4, scores f2, src u1} rewritten atomically per flush."""
def __init__(self, directory):
self.dir = directory
os.makedirs(directory, exist_ok=True)
self.tok_path = os.path.join(directory, "tokens.i16")
self.npz_path = os.path.join(directory, "arrays.npz")
if os.path.exists(self.npz_path):
z = np.load(self.npz_path)
self.lengths = z["lengths"].astype("i4")
self.scores = z["scores"].astype("f2")
self.src = z["src"].astype("u1")
else:
self.lengths = np.zeros(0, dtype="i4")
self.scores = np.zeros(0, dtype="f2")
self.src = np.zeros(0, dtype="u1")
@property
def n_docs(self):
return int(self.lengths.shape[0])
@property
def total_tokens(self):
size = os.path.getsize(self.tok_path) if os.path.exists(self.tok_path) else 0
return size // 2
def append(self, flat, lens, scores, src_ids):
if not lens:
return
with open(self.tok_path, "ab") as f:
f.write(np.asarray(flat, dtype="<i2").tobytes())
self.lengths = np.concatenate([self.lengths, np.asarray(lens, "i4")])
self.scores = np.concatenate([self.scores, np.asarray(scores, "f2")])
self.src = np.concatenate([self.src, np.asarray(src_ids, "u1")])
tmp = self.npz_path + ".tmp.npz"
np.savez(tmp, lengths=self.lengths, scores=self.scores, src=self.src)
os.replace(tmp, self.npz_path)
SRC_IDS = {s["name"]: i for i, s in enumerate(SOURCES)}
def main():
install_deps()
import torch
nm = "cpu"
if torch.cuda.is_available():
nm = torch.cuda.get_device_name(0)
if not LOCAL and ("Tesla T4" not in nm):
raise SystemExit(f"HARDWARE GUARD FAIL: {nm}")
if LOCAL:
print(f"[bootstrap] LOCAL mode: device={nm} (guard bypassed)", flush=True)
else:
print(f"[bootstrap] hardware OK: {nm}", flush=True)
hf = HF()
os.makedirs(OUT, exist_ok=True)
man_local = os.path.join(OUT, "manifest.jsonl")
seen_docs, done_chunks, rows, cum_tokens = 0, set(), [], 0
dl = hf.download(f"{HF_PREFIX}/manifest.jsonl", "/kaggle/tmp/man")
if dl and os.path.exists(dl):
txt = open(dl).read()
for line in txt.splitlines():
if line.strip():
r = json.loads(line)
rows.append(r)
done_chunks.add(r["chunk"])
cum_tokens += int(r["tokens_out"])
seen_docs = max(seen_docs, int(r["seen_end"]))
open(man_local, "w").write(txt)
print(f"[resume] chunks_done={len(done_chunks)} seen={seen_docs} "
f"cum={cum_tokens:,}", flush=True)
from tokenizers import Tokenizer, decoders, models, pre_tokenizers, trainers
tok_path = os.path.join(OUT, "tokenizer.json")
have_tok = os.path.exists(tok_path)
if not have_tok:
got = hf.download(f"{HF_PREFIX}/tokenizer.json", "/kaggle/tmp/tk")
if got and os.path.exists(got):
import shutil
shutil.copyfile(got, tok_path)
have_tok = True
stream, dropped = build_stream()
# breadcrumb IMMEDIATELY after streams resolve — if this lands on HF and
# no heartbeat follows within ~6 min, the stall is between here and loop.
json.dump({"stage": "preprocess", "phase": "streams-loaded", "ts": time.time()},
open(os.path.join(OUT, "hb.json"), "w"))
hf.push_file(os.path.join(OUT, "hb.json"), "heartbeat_preprocess.json")
t0 = time.time()
last_hb = t0
if not have_tok:
print(f"[tok] sampling {TOKENIZER_SAMPLE_DOCS} docs ...", flush=True)
sample, taken = [], 0
for ex in stream:
if taken >= TOKENIZER_SAMPLE_DOCS:
break
taken += 1
_, t = extract_text(ex)
if len(t) >= 200:
sample.append(t[:8000])
tk = Tokenizer(models.BPE(unk_token="<unk>"))
tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False)
tk.decoder = decoders.ByteLevel()
tk.train_from_iterator(sample, trainers.BpeTrainer(
vocab_size=VOCAB, special_tokens=["<pad>", "<unk>", "<bos>", "<eos>"],
min_frequency=2))
tk.save(tok_path)
hf.push_file(tok_path, f"{HF_PREFIX}/tokenizer.json")
seen_docs += taken
print(f"[tok] vocab={tk.get_vocab_size()} pushed; sampled={taken}", flush=True)
else:
tk = Tokenizer.from_file(tok_path)
print(f"[tok] loaded existing ({tk.get_vocab_size()})", flush=True)
corpus = Corpus(OUT)
probe = PplProbe()
exact = ExactDedup()
def encode_batch(texts):
enc = tk.encode_batch(texts, add_special_tokens=False)
keep_idx, lens, flat = [], [], []
for j, e in enumerate(enc):
ids = e.ids
if MIN_LEN <= len(ids) <= MAX_LEN:
keep_idx.append(j)
lens.append(len(ids))
flat.extend(ids)
return keep_idx, lens, flat
cum_tokens = max(cum_tokens, corpus.total_tokens)
chunk_idx = (max(done_chunks) + 1) if done_chunks else 0
if seen_docs:
print(f"[resume] fast-forwarding past {seen_docs} docs ...", flush=True)
skipped = 0
for _ex in stream:
skipped += 1
if skipped >= seen_docs:
break
buf = []
def flush():
nonlocal buf, chunk_idx, cum_tokens
if not buf:
return
items = near_dup_filter(buf)
texts = [it["text"] for it in items]
try:
ppls = probe.ppl_batch(texts)
except Exception as e:
print(f"[ppl] batch failed ({e}); keeping chunk w/o ppl filter", flush=True)
ppls = [(PPL_MIN + PPL_MAX) / 2] * len(texts)
passed = [(it, float(p)) for it, p in zip(items, ppls)
if PPL_MIN <= float(p) <= PPL_MAX]
scored = sorted(((quality_score(it["text"]), p, it) for it, p in passed),
key=lambda x: -x[0])
scored = scored[:max(1, int(len(scored) * QUALITY_KEEP_FRAC))]
final_texts = ["<bos> " + it["text"] + " <eos>" for _, _, it in scored]
# PACK (not discard): docs flow through one id stream separated by
# <eos>; cut fixed 256-token windows so long docs contribute fully.
enc = tk.encode_batch(final_texts, add_special_tokens=False)
eos = tk.token_to_id("<eos>") or 0
stream: list[int] = []
bounds: list[tuple[int, float, int]] = [] # (end_excl, score, src)
pos = 0
for e, (sc, _p, it) in zip(enc, scored):
ids = e.ids
n = len(ids) + 1
stream.extend(ids)
stream.append(eos)
pos += n
bounds.append((pos, sc, SRC_IDS.get(it["src"], 255)))
windows = []
cur = 0
while cur < len(stream):
w = stream[cur:cur + MAX_LEN]
cur += MAX_LEN
if len(w) >= MIN_LEN:
windows.append(w)
def meta_at(start_idx):
acc_end = 0
for end_excl, sc, src in bounds:
acc_end = end_excl
if start_idx < end_excl:
return sc, src
return (bounds[-1][1], bounds[-1][2]) if bounds else (0.0, 255)
lens, flat, sc_out, src_out = [], [], [], []
for wi, w in enumerate(windows):
start = wi * MAX_LEN
sc, src = meta_at(start)
lens.append(len(w))
flat.extend(w)
sc_out.append(sc)
src_out.append(src)
# text sidecar: raw kept docs so ANY tokenizer/model track can consume later
for _j, (_s, _p, it) in enumerate(scored):
pass
with open(os.path.join(OUT, "texts.jsonl"), "a", encoding="utf-8") as tf:
for t in final_texts:
tf.write(json.dumps({"text": t}, ensure_ascii=False) + "\n")
row = {"chunk": chunk_idx, "docs_in": len(buf), "after_dedup": len(items),
"after_ppl": len(passed), "docs_kept": len(enc),
"windows": len(lens), "tokens_out": sum(lens),
"seen_end": seen_docs, "ts": time.time()}
if lens:
corpus.append(flat, lens, sc_out, src_out)
cum_tokens += sum(lens)
row["cum_tokens"] = cum_tokens
rows.append(row)
done_chunks.add(chunk_idx)
with open(man_local, "a") as f:
f.write(json.dumps(row) + "\n")
hf.push_file(man_local, f"{HF_PREFIX}/manifest.jsonl")
print(f"[chunk {chunk_idx}] in={row['docs_in']} dedup={row['after_dedup']} "
f"ppl={row['after_ppl']} wins={row['windows']} tok={row['tokens_out']} "
f"cum={cum_tokens:,}", flush=True)
buf = []
chunk_idx += 1
for ex in stream:
seen_docs += 1
src, t = extract_text(ex)
if t and len(t) >= 200 and exact.check(t):
buf.append({"text": t, "src": src})
if len(buf) >= CHUNK_DOCS:
flush()
if corpus.total_tokens >= TARGET_TOKENS:
print(f"[target] hit {TARGET_TOKENS:,}", flush=True)
break
now = time.time()
if now - last_hb >= 300:
hb = {"stage": "preprocess", "seen": seen_docs,
"corpus_tokens": corpus.total_tokens, "corpus_docs": corpus.n_docs,
"ts": now}
p = os.path.join(OUT, "hb.json")
json.dump(hb, open(p, "w"))
hf.push_file(p, "heartbeat_preprocess.json")
last_hb = now
if MAX_MINUTES and (now - t0) >= MAX_MINUTES * 60:
flush()
print("[exit] MAX_MINUTES reached; graceful stop WITHOUT DONE", flush=True)
return
flush()
exhausted = corpus.total_tokens < TARGET_TOKENS
meta = {"vocab": VOCAB, "min_len": MIN_LEN, "max_len": MAX_LEN,
"total_tokens": corpus.total_tokens, "docs": corpus.n_docs,
"target_tokens": TARGET_TOKENS, "exhausted_stream": exhausted,
"tokenizer_sha": sha_bytes(open(tok_path, "rb").read()), "seed": SEED}
json.dump(meta, open(os.path.join(OUT, "corpus_meta.json"), "w"), indent=2)
hf.push_file(os.path.join(OUT, "corpus_meta.json"), f"{HF_PREFIX}/corpus_meta.json")
per_src = {}
for sid in np.unique(corpus.src):
nm_ = next((s["name"] for s, i in SRC_IDS.items() if i == sid), f"unk{sid}")
mask = corpus.src == sid
per_src[str(nm_)] = {"docs": int(mask.sum()),
"tokens": int(corpus.lengths[mask].sum(dtype="i8"))}
stats = {"final": meta, "per_source": per_src, "dropped_sources": dropped,
"chunks": len(rows), "wall_hours": round((time.time() - t0) / 3600, 2)}
sp = "/kaggle/work/preprocess_stats.json"
json.dump(stats, open(sp, "w"))
hf.push_file(sp, "telemetry/preprocess_stats.json")
do_publish = PUBLISH and not LOCAL
published = publish_kaggle_dataset(OUT, user_creds()) if do_publish else False
if not published:
print("[publish] Kaggle dataset NOT created (LOCAL mode or creds missing); corpus stays in kernel output.", flush=True)
with open(os.path.join(OUT, "DONE"), "w") as f:
f.write(json.dumps(meta))
hf.push_file(os.path.join(OUT, "DONE"), f"{HF_PREFIX}/DONE")
print(f"[done] tokens={corpus.total_tokens:,} docs={corpus.n_docs} "
f"published={published} exhausted={exhausted}", flush=True)
def user_creds():
try:
from kaggle_secrets import UserSecretsClient
c = UserSecretsClient()
return c.get_secret("KAGGLE_USERNAME"), c.get_secret("KAGGLE_API_TOKEN")
except Exception:
return None, None
def publish_kaggle_dataset(folder, creds):
user, key = creds
if not (user and key):
print("[publish] no Kaggle creds in secrets; skipping", flush=True)
return False
subprocess.run([sys.executable, "-m", "pip", "install", "-q", "kaggle"], check=True)
env = dict(os.environ, KAGGLE_USERNAME=user, KAGGLE_KEY=key)
json.dump({"title": "ae-corpus-v1", "id": f"{user}/ae-corpus-v1",
"licenses": [{"name": "CC0-1.0"}]},
open(os.path.join(folder, "dataset-metadata.json"), "w"))
for cmd, label in (
([sys.executable, "-m", "kaggle", "datasets", "create", "-p", folder,
"--dir-mode", "zip"], "create"),
([sys.executable, "-m", "kaggle", "datasets", "version", "-p", folder,
"--dir-mode", "zip", "-m", "update"], "version"),
):
r = subprocess.run(cmd, env=env, capture_output=True, text=True)
tail = ((r.stdout or "") + (r.stderr or ""))[-300:]
print(f"[publish] {label} rc={r.returncode} {tail}", flush=True)
if r.returncode == 0:
return True
return False
if __name__ == "__main__":
main()