Download benchmarks/get_deep1m.py from Nabidnur/PISCES: direct link, hf CLI and curl.
- Browser
- Download file 3.16 kB
-
https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/get_deep1m.py
- Command line
-
hf download hf://Nabidnur/PISCES/benchmarks/get_deep1m.py
-
curl -L -o get_deep1m.py https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/get_deep1m.py
3.16 kB
| """Download DEEP* 1M exactly as the e-means paper uses it, and verify it. | |
| Fetches bytes [0, 8 + 1,010,000 * 96 * 4) of Deep1B's base.1B.fbin from the Hugging Face mirror the | |
| paper's download script uses (suchun/deep1b-mirror), takes the first 1M rows as train and the next | |
| 10,000 as the evaluation set, splits that set with permutation seed 1234 (first half validation, | |
| second half test), and checks all three against the SHA-256 fingerprints in faiss's | |
| benchs/e_means/paper_splits/fingerprints.json (copied below). Writes train_1M.npy and test.npy. | |
| Usage: python benchmarks/get_deep1m.py DATA_DIR | |
| """ | |
| import hashlib, os, sys, urllib.request | |
| import numpy as np | |
| URL = "https://huggingface.co/datasets/suchun/deep1b-mirror/resolve/main/base.1B.fbin" | |
| N, EVAL, D = 1_000_000, 10_000, 96 | |
| FINGERPRINTS = { # faiss benchs/e_means/paper_splits/fingerprints.json, entry "deep1b-1M" | |
| "val": "ca94deceddb13e16d6cfb81365e2e18c319bf704b1d0ea9a28e5067bba03c458", "test": "05d685ccf5e413a0559348b3586ec82a72da637d50b7d2b9a521e70c6cd29b6f", "train_rows": "7b74aeede93a64c6a2e598f021fbaa42061a7e09e9f3f75f1c1254fbb4419368"} | |
| def sha(a): | |
| return hashlib.sha256(np.ascontiguousarray(a, dtype=np.float32).tobytes()).hexdigest() | |
| def main(out): | |
| os.makedirs(out, exist_ok=True) | |
| files = [os.path.join(out, f) for f in ("train_1M.npy", "val.npy", "test.npy")] | |
| if all(os.path.exists(f) for f in files): # already prepared: verify instead of downloading | |
| train, val, test = (np.load(f, mmap_mode="r") for f in files) | |
| sample = np.linspace(0, N - 1, 1000).astype(np.int64) | |
| got = {"val": sha(val), "test": sha(test), "train_rows": sha(np.array(train[sample]))} | |
| if all(got[key] == want for key, want in FINGERPRINTS.items()): | |
| print("DEEP* 1M already in %s; fingerprints match the paper's" % out) | |
| return | |
| raw = os.path.join(out, "deep1b_prefix.fbin") | |
| nbytes = 8 + (N + EVAL) * D * 4 | |
| if not os.path.exists(raw) or os.path.getsize(raw) != nbytes: | |
| req = urllib.request.Request(URL, headers={"Range": "bytes=0-%d" % (nbytes - 1)}) | |
| with urllib.request.urlopen(req, timeout=600) as r, open(raw, "wb") as f: | |
| while True: | |
| b = r.read(1 << 24) | |
| if not b: | |
| break | |
| f.write(b) | |
| m = np.memmap(raw, dtype=np.float32, mode="r", offset=8, shape=(N + EVAL, D)) | |
| train, ev = m[:N], np.array(m[N:], dtype=np.float32) | |
| perm = np.random.RandomState(1234).permutation(EVAL) | |
| val, test = ev[perm[:EVAL // 2]], ev[perm[EVAL // 2:]] | |
| sample = np.linspace(0, N - 1, 1000).astype(np.int64) | |
| got = {"val": sha(val), "test": sha(test), "train_rows": sha(np.array(train[sample]))} | |
| for key, want in FINGERPRINTS.items(): | |
| assert got[key] == want, "%s rows differ from the paper's" % key | |
| np.save(os.path.join(out, "train_1M.npy"), np.array(train, dtype=np.float32)) | |
| np.save(os.path.join(out, "val.npy"), val) | |
| np.save(os.path.join(out, "test.npy"), test) | |
| os.remove(raw) | |
| print("DEEP* 1M ready in %s; fingerprints match the paper's" % out) | |
| if __name__ == "__main__": | |
| main(sys.argv[1]) | |