File size: 3,162 Bytes
c09a3f4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 | """Download DEEP* 1M exactly as the e-means paper uses it, and verify it.
Fetches bytes [0, 8 + 1,010,000 * 96 * 4) of Deep1B's base.1B.fbin from the Hugging Face mirror the
paper's download script uses (suchun/deep1b-mirror), takes the first 1M rows as train and the next
10,000 as the evaluation set, splits that set with permutation seed 1234 (first half validation,
second half test), and checks all three against the SHA-256 fingerprints in faiss's
benchs/e_means/paper_splits/fingerprints.json (copied below). Writes train_1M.npy and test.npy.
Usage: python benchmarks/get_deep1m.py DATA_DIR
"""
import hashlib, os, sys, urllib.request
import numpy as np
URL = "https://huggingface.co/datasets/suchun/deep1b-mirror/resolve/main/base.1B.fbin"
N, EVAL, D = 1_000_000, 10_000, 96
FINGERPRINTS = { # faiss benchs/e_means/paper_splits/fingerprints.json, entry "deep1b-1M"
"val": "ca94deceddb13e16d6cfb81365e2e18c319bf704b1d0ea9a28e5067bba03c458", "test": "05d685ccf5e413a0559348b3586ec82a72da637d50b7d2b9a521e70c6cd29b6f", "train_rows": "7b74aeede93a64c6a2e598f021fbaa42061a7e09e9f3f75f1c1254fbb4419368"}
def sha(a):
return hashlib.sha256(np.ascontiguousarray(a, dtype=np.float32).tobytes()).hexdigest()
def main(out):
os.makedirs(out, exist_ok=True)
files = [os.path.join(out, f) for f in ("train_1M.npy", "val.npy", "test.npy")]
if all(os.path.exists(f) for f in files): # already prepared: verify instead of downloading
train, val, test = (np.load(f, mmap_mode="r") for f in files)
sample = np.linspace(0, N - 1, 1000).astype(np.int64)
got = {"val": sha(val), "test": sha(test), "train_rows": sha(np.array(train[sample]))}
if all(got[key] == want for key, want in FINGERPRINTS.items()):
print("DEEP* 1M already in %s; fingerprints match the paper's" % out)
return
raw = os.path.join(out, "deep1b_prefix.fbin")
nbytes = 8 + (N + EVAL) * D * 4
if not os.path.exists(raw) or os.path.getsize(raw) != nbytes:
req = urllib.request.Request(URL, headers={"Range": "bytes=0-%d" % (nbytes - 1)})
with urllib.request.urlopen(req, timeout=600) as r, open(raw, "wb") as f:
while True:
b = r.read(1 << 24)
if not b:
break
f.write(b)
m = np.memmap(raw, dtype=np.float32, mode="r", offset=8, shape=(N + EVAL, D))
train, ev = m[:N], np.array(m[N:], dtype=np.float32)
perm = np.random.RandomState(1234).permutation(EVAL)
val, test = ev[perm[:EVAL // 2]], ev[perm[EVAL // 2:]]
sample = np.linspace(0, N - 1, 1000).astype(np.int64)
got = {"val": sha(val), "test": sha(test), "train_rows": sha(np.array(train[sample]))}
for key, want in FINGERPRINTS.items():
assert got[key] == want, "%s rows differ from the paper's" % key
np.save(os.path.join(out, "train_1M.npy"), np.array(train, dtype=np.float32))
np.save(os.path.join(out, "val.npy"), val)
np.save(os.path.join(out, "test.npy"), test)
os.remove(raw)
print("DEEP* 1M ready in %s; fingerprints match the paper's" % out)
if __name__ == "__main__":
main(sys.argv[1])
|