"""Download DEEP* 1M exactly as the e-means paper uses it, and verify it. Fetches bytes [0, 8 + 1,010,000 * 96 * 4) of Deep1B's base.1B.fbin from the Hugging Face mirror the paper's download script uses (suchun/deep1b-mirror), takes the first 1M rows as train and the next 10,000 as the evaluation set, splits that set with permutation seed 1234 (first half validation, second half test), and checks all three against the SHA-256 fingerprints in faiss's benchs/e_means/paper_splits/fingerprints.json (copied below). Writes train_1M.npy and test.npy. Usage: python benchmarks/get_deep1m.py DATA_DIR """ import hashlib, os, sys, urllib.request import numpy as np URL = "https://huggingface.co/datasets/suchun/deep1b-mirror/resolve/main/base.1B.fbin" N, EVAL, D = 1_000_000, 10_000, 96 FINGERPRINTS = { # faiss benchs/e_means/paper_splits/fingerprints.json, entry "deep1b-1M" "val": "ca94deceddb13e16d6cfb81365e2e18c319bf704b1d0ea9a28e5067bba03c458", "test": "05d685ccf5e413a0559348b3586ec82a72da637d50b7d2b9a521e70c6cd29b6f", "train_rows": "7b74aeede93a64c6a2e598f021fbaa42061a7e09e9f3f75f1c1254fbb4419368"} def sha(a): return hashlib.sha256(np.ascontiguousarray(a, dtype=np.float32).tobytes()).hexdigest() def main(out): os.makedirs(out, exist_ok=True) files = [os.path.join(out, f) for f in ("train_1M.npy", "val.npy", "test.npy")] if all(os.path.exists(f) for f in files): # already prepared: verify instead of downloading train, val, test = (np.load(f, mmap_mode="r") for f in files) sample = np.linspace(0, N - 1, 1000).astype(np.int64) got = {"val": sha(val), "test": sha(test), "train_rows": sha(np.array(train[sample]))} if all(got[key] == want for key, want in FINGERPRINTS.items()): print("DEEP* 1M already in %s; fingerprints match the paper's" % out) return raw = os.path.join(out, "deep1b_prefix.fbin") nbytes = 8 + (N + EVAL) * D * 4 if not os.path.exists(raw) or os.path.getsize(raw) != nbytes: req = urllib.request.Request(URL, headers={"Range": "bytes=0-%d" % (nbytes - 1)}) with urllib.request.urlopen(req, timeout=600) as r, open(raw, "wb") as f: while True: b = r.read(1 << 24) if not b: break f.write(b) m = np.memmap(raw, dtype=np.float32, mode="r", offset=8, shape=(N + EVAL, D)) train, ev = m[:N], np.array(m[N:], dtype=np.float32) perm = np.random.RandomState(1234).permutation(EVAL) val, test = ev[perm[:EVAL // 2]], ev[perm[EVAL // 2:]] sample = np.linspace(0, N - 1, 1000).astype(np.int64) got = {"val": sha(val), "test": sha(test), "train_rows": sha(np.array(train[sample]))} for key, want in FINGERPRINTS.items(): assert got[key] == want, "%s rows differ from the paper's" % key np.save(os.path.join(out, "train_1M.npy"), np.array(train, dtype=np.float32)) np.save(os.path.join(out, "val.npy"), val) np.save(os.path.join(out, "test.npy"), test) os.remove(raw) print("DEEP* 1M ready in %s; fingerprints match the paper's" % out) if __name__ == "__main__": main(sys.argv[1])