"""Stage 3 preprocess kernel v2: streaming mix -> dedup -> perplexity filter -> quality top-80% -> BPE-8192 tokenize -> packed numpy corpus -> Kaggle dataset ae-corpus-v1 (private). Self-contained. Resumable via HF-mirrored manifest. v2 fixes (2026-08-24, from first-run log): - ultrachat split train_sft (train doesn't exist) - synthcot source swapped to Open-Orca/SlimOrca (open-phi/synthetic-cot DNE) - stack_py: HF token passed to load_dataset (gated repo); still requires the account to have accepted bigcode terms once on the Hub - else auto-dropped - storage: zarr-v3 removed Group.create_dataset -> replaced with dependency- free append-only tokens.i16 + atomic arrays.npz Outputs: /kaggle/work/corpus/ tokens.i16, arrays.npz {lengths,scores,src}, tokenizer.json, corpus_meta.json HF preprocess/manifest.jsonl, preprocess/tokenizer.json, telemetry/preprocess_stats.json, preprocess/DONE Secret aliases required: HF_TOKEN, KAGGLE_USERNAME, KAGGLE_API_TOKEN. """ import hashlib import json import os import subprocess import sys import time import numpy as np REPO = os.environ.get("HF_CKPT_REPO", "geraldaton20/student-ckpts-v1") OUT = os.environ.get("AE_OUT", "/kaggle/work/corpus") LOCAL = os.environ.get("AE_LOCAL", "0") == "1" PUBLISH = os.environ.get("AE_PUBLISH", "" ) != "0" HF_PREFIX = "preprocess" TARGET_TOKENS = int(os.environ.get("TARGET_TOKENS", "850000000")) CHUNK_DOCS = int(os.environ.get("CHUNK_DOCS", "20000")) TOKENIZER_SAMPLE_DOCS = int(os.environ.get("TOKENIZER_SAMPLE_DOCS", "60000")) VOCAB = 8192 MIN_LEN, MAX_LEN = 50, 256 PPL_MIN, PPL_MAX = 5.0, 50.0 PROBE_MODEL = "gpt2" QUALITY_KEEP_FRAC = 0.80 MAX_MINUTES = int(os.environ.get("MAX_MINUTES", "360")) SEED = 42 SOURCES = [ {"name": "cosmopedia", "repo": "HuggingFaceTB/cosmopedia", "config": "stanford", "ratio": 0.40, "split": "train"}, {"name": "stack_py", "repo": "bigcode/the-stack-v2", "config": None, "ratio": 0.30, "field": "content", "split": "train", "gated": True, # non-gated fallback so the pipeline never blocks on token scopes: "alt": {"repo": "code_search_net", "config": "python", "ratio": 0.30, "field": "func_code_string", "split": "train"}}, {"name": "ultrachat", "repo": "openbmb/UltraChat", "config": None, "ratio": 0.20, "field": "data", "split": "train"}, {"name": "synthcot", "repo": "casperhansen/gsm8k_synthetic_cot", "config": None, "ratio": 0.10, "split": "train", "join_fields": ["question", "cot", "answer"]}, ] # SPEC CORRECTIONS 2026-08-24: open-phi/synthetic-cot does not exist -> # casperhansen/gsm8k_synthetic_cot (user-selected); ultrachat moved to the # original openbmb/UltraChat (single 'train' split, 'data' turn list). def install_deps(): missing = [] for m in ("datasets", "transformers", "tokenizers", "huggingface_hub"): try: __import__(m) except Exception: missing.append(m) if missing: subprocess.run([sys.executable, "-m", "pip", "install", "-q", *missing], check=True) print("[bootstrap] deps ok", flush=True) def hf_token(): tok = os.environ.get("HF_TOKEN") if tok: return tok try: from kaggle_secrets import UserSecretsClient return UserSecretsClient().get_secret("HF_TOKEN") except Exception: return None class HF: def __init__(self): self.token = hf_token() from huggingface_hub import HfApi self.api = HfApi(token=self.token) if self.token else None def push_file(self, local, path_in_repo): if not self.api: return last = None for i in range(4): try: self.api.upload_file(path_or_fileobj=local, path_in_repo=path_in_repo, repo_id=REPO, repo_type="model") return except Exception as e: last = e time.sleep(2 * (i + 1)) print(f"[hf] push {path_in_repo} failed: {last}", flush=True) def download(self, path_in_repo, local_dir): if not self.api: return None try: return self.api.hf_hub_download(repo_id=REPO, filename=path_in_repo, repo_type="model", local_dir=local_dir) except Exception: return None def sha_bytes(b): return hashlib.sha256(b).hexdigest()[:16] def extract_text(ex): src = ex.get("__src", "") spec = next((s for s in SOURCES if s["name"] == src), {}) if spec.get("join_fields"): parts = [] for f in spec["join_fields"]: v = ex.get(f) if isinstance(v, list): parts.extend(x for x in v if isinstance(x, str)) elif isinstance(v, str): parts.append(v) if parts: return src, "\n".join(parts) f = spec.get("field") if f and f in ex: v = ex[f] if isinstance(v, str): return src, v if isinstance(v, list): out = [] for m in v: if isinstance(m, str): out.append(m) elif isinstance(m, dict): for k in ("content", "text", "value"): if isinstance(m.get(k), str): out.append(m[k]) break return src, "\n".join(out) for k in ("text", "content", "question", "instruction", "prompt"): v = ex.get(k) if isinstance(v, str) and v.strip(): return src, v return src, "" def build_stream(): """Load every source (with optional non-gated fallbacks) and interleave.""" from datasets import interleave_datasets, load_dataset def try_one(spec): kw = dict(path=spec["repo"], split=spec["split"], streaming=True, token=hf_token()) if spec.get("config"): kw["name"] = spec["config"] return load_dataset(**kw) sets, probs, dropped = [], [], [] for spec in SOURCES: candidates = [spec] + ([spec["alt"]] if spec.get("alt") else []) loaded = None for cand in candidates: try: ds = try_one(cand) loaded = (ds, cand) tag = "" if cand is spec else " [fallback]" print(f"[stream] {spec['name']}{tag} OK ({cand['ratio']:.0%})", flush=True) break except Exception as e: hint = "" if cand.get("gated") and "gated" in str(e).lower(): hint = (" (gated: token must cover this repo - a CLASSIC " "token carries account grants; fine-grained needs " f"the repo explicitly selected)") print(f"[stream] try {cand['repo']} FAILED ({e}){hint}", flush=True) if loaded is None: dropped.append(spec["name"]) continue ds, cand = loaded # merge the winning candidate's extraction hints onto the source spec for k in ("field", "join_fields", "config"): if k in cand: spec[k] = cand[k] name = spec["name"] ds = ds.map(lambda ex, _n=name: {"__src": _n}) sets.append(ds) probs.append(spec["ratio"]) if not sets: raise SystemExit("no sources loaded") tot = sum(probs) probs = [p / tot for p in probs] mixed = interleave_datasets(sets, probabilities=probs, seed=SEED) return mixed, dropped class ExactDedup: def __init__(self, cap=20_000_000): self.seen = set() self.cap = cap def check(self, text): h = int.from_bytes(hashlib.blake2b( " ".join(text.lower().split())[:4000].encode(), digest_size=8).digest(), "little") if h in self.seen: return False if len(self.seen) < self.cap: self.seen.add(h) return True def near_dup_filter(items): """Per-chunk MinHash LSH near-dup removal (chunk-scoped, idempotent).""" try: from datasketch import MinHash, MinHashLSH except Exception: return items lsh = MinHashLSH(threshold=0.85, num_perm=128) out = [] for j, it in enumerate(items): words = it["text"].split() m = MinHash(num_perm=128) n = 0 for s in range(0, max(1, len(words) - 4), 3): m.update(" ".join(words[s:s + 5]).encode("utf8")) n += 1 if n >= 200: break if n == 0: out.append(it) continue dup = False try: for _key in lsh.query(m): dup = True break except Exception: pass if not dup: try: lsh.insert(str(j), m) except Exception: pass out.append(it) return out def quality_score(text): score = 0.0 if any(kw in text for kw in ("def ", "class ", "import ", "function")): score += 0.3 if any(marker in text for marker in ("Therefore,", "Step 1:", "First,", "Because")): score += 0.2 toks = len(text.split()) if toks < 30: score -= 0.3 elif toks > 200: score += 0.1 uniq = len(set(text.split())) / max(1, len(text.split())) return max(0.0, min(1.0, score + uniq * 0.2)) class PplProbe: def __init__(self): import torch from transformers import AutoModelForCausalLM, AutoTokenizer self.torch = torch self.dev = "cuda" if torch.cuda.is_available() else "cpu" self.tk = AutoTokenizer.from_pretrained(PROBE_MODEL) if self.tk.pad_token_id is None: self.tk.pad_token = self.tk.eos_token dt = torch.float16 if self.dev == "cuda" else torch.float32 self.model = AutoModelForCausalLM.from_pretrained( PROBE_MODEL, torch_dtype=dt).to(self.dev).eval() def ppl_batch(self, texts, batch=32): torch = self.torch out = [] with torch.no_grad(): for s in range(0, len(texts), batch): part = [t[:1500] for t in texts[s:s + batch]] enc = self.tk(part, return_tensors="pt", padding=True, truncation=True, max_length=256).to(self.dev) logits = self.model(**enc).logits.float() labels = enc["input_ids"][:, 1:] mask = enc["attention_mask"][:, 1:].float() nll = torch.nn.functional.cross_entropy( logits[:, :-1].reshape(-1, logits.size(-1)), labels.reshape(-1), reduction="none").view(labels.shape) nll = (nll * mask).sum(1) / mask.sum(1).clamp(min=1) out.extend(torch.exp(nll).tolist()) return out class Corpus: """Dependency-free corpus store: tokens.i16 (append-only, int16 LE) + arrays.npz {lengths i4, scores f2, src u1} rewritten atomically per flush.""" def __init__(self, directory): self.dir = directory os.makedirs(directory, exist_ok=True) self.tok_path = os.path.join(directory, "tokens.i16") self.npz_path = os.path.join(directory, "arrays.npz") if os.path.exists(self.npz_path): z = np.load(self.npz_path) self.lengths = z["lengths"].astype("i4") self.scores = z["scores"].astype("f2") self.src = z["src"].astype("u1") else: self.lengths = np.zeros(0, dtype="i4") self.scores = np.zeros(0, dtype="f2") self.src = np.zeros(0, dtype="u1") @property def n_docs(self): return int(self.lengths.shape[0]) @property def total_tokens(self): size = os.path.getsize(self.tok_path) if os.path.exists(self.tok_path) else 0 return size // 2 def append(self, flat, lens, scores, src_ids): if not lens: return with open(self.tok_path, "ab") as f: f.write(np.asarray(flat, dtype="= TOKENIZER_SAMPLE_DOCS: break taken += 1 _, t = extract_text(ex) if len(t) >= 200: sample.append(t[:8000]) tk = Tokenizer(models.BPE(unk_token="")) tk.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) tk.decoder = decoders.ByteLevel() tk.train_from_iterator(sample, trainers.BpeTrainer( vocab_size=VOCAB, special_tokens=["", "", "", ""], min_frequency=2)) tk.save(tok_path) hf.push_file(tok_path, f"{HF_PREFIX}/tokenizer.json") seen_docs += taken print(f"[tok] vocab={tk.get_vocab_size()} pushed; sampled={taken}", flush=True) else: tk = Tokenizer.from_file(tok_path) print(f"[tok] loaded existing ({tk.get_vocab_size()})", flush=True) corpus = Corpus(OUT) probe = PplProbe() exact = ExactDedup() def encode_batch(texts): enc = tk.encode_batch(texts, add_special_tokens=False) keep_idx, lens, flat = [], [], [] for j, e in enumerate(enc): ids = e.ids if MIN_LEN <= len(ids) <= MAX_LEN: keep_idx.append(j) lens.append(len(ids)) flat.extend(ids) return keep_idx, lens, flat cum_tokens = max(cum_tokens, corpus.total_tokens) chunk_idx = (max(done_chunks) + 1) if done_chunks else 0 if seen_docs: print(f"[resume] fast-forwarding past {seen_docs} docs ...", flush=True) skipped = 0 for _ex in stream: skipped += 1 if skipped >= seen_docs: break buf = [] def flush(): nonlocal buf, chunk_idx, cum_tokens if not buf: return items = near_dup_filter(buf) texts = [it["text"] for it in items] try: ppls = probe.ppl_batch(texts) except Exception as e: print(f"[ppl] batch failed ({e}); keeping chunk w/o ppl filter", flush=True) ppls = [(PPL_MIN + PPL_MAX) / 2] * len(texts) passed = [(it, float(p)) for it, p in zip(items, ppls) if PPL_MIN <= float(p) <= PPL_MAX] scored = sorted(((quality_score(it["text"]), p, it) for it, p in passed), key=lambda x: -x[0]) scored = scored[:max(1, int(len(scored) * QUALITY_KEEP_FRAC))] final_texts = [" " + it["text"] + " " for _, _, it in scored] # PACK (not discard): docs flow through one id stream separated by # ; cut fixed 256-token windows so long docs contribute fully. enc = tk.encode_batch(final_texts, add_special_tokens=False) eos = tk.token_to_id("") or 0 stream: list[int] = [] bounds: list[tuple[int, float, int]] = [] # (end_excl, score, src) pos = 0 for e, (sc, _p, it) in zip(enc, scored): ids = e.ids n = len(ids) + 1 stream.extend(ids) stream.append(eos) pos += n bounds.append((pos, sc, SRC_IDS.get(it["src"], 255))) windows = [] cur = 0 while cur < len(stream): w = stream[cur:cur + MAX_LEN] cur += MAX_LEN if len(w) >= MIN_LEN: windows.append(w) def meta_at(start_idx): acc_end = 0 for end_excl, sc, src in bounds: acc_end = end_excl if start_idx < end_excl: return sc, src return (bounds[-1][1], bounds[-1][2]) if bounds else (0.0, 255) lens, flat, sc_out, src_out = [], [], [], [] for wi, w in enumerate(windows): start = wi * MAX_LEN sc, src = meta_at(start) lens.append(len(w)) flat.extend(w) sc_out.append(sc) src_out.append(src) # text sidecar: raw kept docs so ANY tokenizer/model track can consume later for _j, (_s, _p, it) in enumerate(scored): pass with open(os.path.join(OUT, "texts.jsonl"), "a", encoding="utf-8") as tf: for t in final_texts: tf.write(json.dumps({"text": t}, ensure_ascii=False) + "\n") row = {"chunk": chunk_idx, "docs_in": len(buf), "after_dedup": len(items), "after_ppl": len(passed), "docs_kept": len(enc), "windows": len(lens), "tokens_out": sum(lens), "seen_end": seen_docs, "ts": time.time()} if lens: corpus.append(flat, lens, sc_out, src_out) cum_tokens += sum(lens) row["cum_tokens"] = cum_tokens rows.append(row) done_chunks.add(chunk_idx) with open(man_local, "a") as f: f.write(json.dumps(row) + "\n") hf.push_file(man_local, f"{HF_PREFIX}/manifest.jsonl") print(f"[chunk {chunk_idx}] in={row['docs_in']} dedup={row['after_dedup']} " f"ppl={row['after_ppl']} wins={row['windows']} tok={row['tokens_out']} " f"cum={cum_tokens:,}", flush=True) buf = [] chunk_idx += 1 for ex in stream: seen_docs += 1 src, t = extract_text(ex) if t and len(t) >= 200 and exact.check(t): buf.append({"text": t, "src": src}) if len(buf) >= CHUNK_DOCS: flush() if corpus.total_tokens >= TARGET_TOKENS: print(f"[target] hit {TARGET_TOKENS:,}", flush=True) break now = time.time() if now - last_hb >= 300: hb = {"stage": "preprocess", "seen": seen_docs, "corpus_tokens": corpus.total_tokens, "corpus_docs": corpus.n_docs, "ts": now} p = os.path.join(OUT, "hb.json") json.dump(hb, open(p, "w")) hf.push_file(p, "heartbeat_preprocess.json") last_hb = now if MAX_MINUTES and (now - t0) >= MAX_MINUTES * 60: flush() print("[exit] MAX_MINUTES reached; graceful stop WITHOUT DONE", flush=True) return flush() exhausted = corpus.total_tokens < TARGET_TOKENS meta = {"vocab": VOCAB, "min_len": MIN_LEN, "max_len": MAX_LEN, "total_tokens": corpus.total_tokens, "docs": corpus.n_docs, "target_tokens": TARGET_TOKENS, "exhausted_stream": exhausted, "tokenizer_sha": sha_bytes(open(tok_path, "rb").read()), "seed": SEED} json.dump(meta, open(os.path.join(OUT, "corpus_meta.json"), "w"), indent=2) hf.push_file(os.path.join(OUT, "corpus_meta.json"), f"{HF_PREFIX}/corpus_meta.json") per_src = {} for sid in np.unique(corpus.src): nm_ = next((s["name"] for s, i in SRC_IDS.items() if i == sid), f"unk{sid}") mask = corpus.src == sid per_src[str(nm_)] = {"docs": int(mask.sum()), "tokens": int(corpus.lengths[mask].sum(dtype="i8"))} stats = {"final": meta, "per_source": per_src, "dropped_sources": dropped, "chunks": len(rows), "wall_hours": round((time.time() - t0) / 3600, 2)} sp = "/kaggle/work/preprocess_stats.json" json.dump(stats, open(sp, "w")) hf.push_file(sp, "telemetry/preprocess_stats.json") do_publish = PUBLISH and not LOCAL published = publish_kaggle_dataset(OUT, user_creds()) if do_publish else False if not published: print("[publish] Kaggle dataset NOT created (LOCAL mode or creds missing); corpus stays in kernel output.", flush=True) with open(os.path.join(OUT, "DONE"), "w") as f: f.write(json.dumps(meta)) hf.push_file(os.path.join(OUT, "DONE"), f"{HF_PREFIX}/DONE") print(f"[done] tokens={corpus.total_tokens:,} docs={corpus.n_docs} " f"published={published} exhausted={exhausted}", flush=True) def user_creds(): try: from kaggle_secrets import UserSecretsClient c = UserSecretsClient() return c.get_secret("KAGGLE_USERNAME"), c.get_secret("KAGGLE_API_TOKEN") except Exception: return None, None def publish_kaggle_dataset(folder, creds): user, key = creds if not (user and key): print("[publish] no Kaggle creds in secrets; skipping", flush=True) return False subprocess.run([sys.executable, "-m", "pip", "install", "-q", "kaggle"], check=True) env = dict(os.environ, KAGGLE_USERNAME=user, KAGGLE_KEY=key) json.dump({"title": "ae-corpus-v1", "id": f"{user}/ae-corpus-v1", "licenses": [{"name": "CC0-1.0"}]}, open(os.path.join(folder, "dataset-metadata.json"), "w")) for cmd, label in ( ([sys.executable, "-m", "kaggle", "datasets", "create", "-p", folder, "--dir-mode", "zip"], "create"), ([sys.executable, "-m", "kaggle", "datasets", "version", "-p", folder, "--dir-mode", "zip", "-m", "update"], "version"), ): r = subprocess.run(cmd, env=env, capture_output=True, text=True) tail = ((r.stdout or "") + (r.stderr or ""))[-300:] print(f"[publish] {label} rc={r.returncode} {tail}", flush=True) if r.returncode == 0: return True return False if __name__ == "__main__": main()