File size: 2,720 Bytes
753bfab | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | #!/usr/bin/env python3
"""Fetch WikiText-103-raw and write it as a flat character corpus.
WHY A STANDARD CORPUS IS THE POINT
Every number measured so far is on 683,065 characters of Cory's own logs. That corpus
cannot answer the scaling question for two separate reasons:
1. At 10M+ parameters a 683K-character corpus is memorised, so every rung converges to
the same overfit floor and the comparison silently becomes about regularisation
rather than architecture.
2. Results on a private corpus are not checkable by anyone else. WikiText-103 is the
benchmark the field already uses, so a dyn12 advantage measured here is directly
comparable to published work instead of being a claim about one person's log files.
One train shard is ~157 MB of parquet, which yields roughly a quarter of a billion
characters -- enough that a 30M-parameter model is data-limited rather than
memorisation-limited.
python tools/fetch_wikitext.py [--shards 1]
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.stdout.reconfigure(encoding="utf-8", errors="replace")
OUT = Path("01_HER_SOUL/corpus_snapshots/wikitext103_train.txt")
REPO = "Salesforce/wikitext"
SHARDS = ["wikitext-103-raw-v1/train-00000-of-00002.parquet",
"wikitext-103-raw-v1/train-00001-of-00002.parquet"]
def main() -> int:
n = 1
if "--shards" in sys.argv:
n = int(sys.argv[sys.argv.index("--shards") + 1])
import pyarrow.parquet as pq
from huggingface_hub import hf_hub_download
OUT.parent.mkdir(parents=True, exist_ok=True)
total = 0
with OUT.open("w", encoding="utf-8", newline="\n") as f:
for s in SHARDS[:n]:
print(f" downloading {s} ...", flush=True)
local = hf_hub_download(REPO, s, repo_type="dataset")
t = pq.read_table(local)
col = t.column("text").to_pylist()
# WikiText ships one row per line, blanks and " = Heading = " markers included.
# Both are kept: they carry document structure a character model can learn.
for line in col:
if line:
f.write(line)
total += len(line)
print(f" {len(col):,} rows, running total {total/1e6:.1f}M chars", flush=True)
size = OUT.stat().st_size
print(f"\n wrote {OUT}")
print(f" {total:,} characters, {size/1e6:.1f} MB on disk")
import hashlib
h = hashlib.sha256()
with OUT.open("rb") as fh:
for c in iter(lambda: fh.read(1 << 20), b""):
h.update(c)
print(f" sha256 {h.hexdigest()[:16]} <- freeze this in any result table")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|