File size: 3,162 Bytes
c09a3f4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
"""Download DEEP* 1M exactly as the e-means paper uses it, and verify it.

Fetches bytes [0, 8 + 1,010,000 * 96 * 4) of Deep1B's base.1B.fbin from the Hugging Face mirror the
paper's download script uses (suchun/deep1b-mirror), takes the first 1M rows as train and the next
10,000 as the evaluation set, splits that set with permutation seed 1234 (first half validation,
second half test), and checks all three against the SHA-256 fingerprints in faiss's
benchs/e_means/paper_splits/fingerprints.json (copied below). Writes train_1M.npy and test.npy.

Usage: python benchmarks/get_deep1m.py DATA_DIR
"""
import hashlib, os, sys, urllib.request
import numpy as np

URL = "https://huggingface.co/datasets/suchun/deep1b-mirror/resolve/main/base.1B.fbin"
N, EVAL, D = 1_000_000, 10_000, 96
FINGERPRINTS = {  # faiss benchs/e_means/paper_splits/fingerprints.json, entry "deep1b-1M"
    "val": "ca94deceddb13e16d6cfb81365e2e18c319bf704b1d0ea9a28e5067bba03c458", "test": "05d685ccf5e413a0559348b3586ec82a72da637d50b7d2b9a521e70c6cd29b6f", "train_rows": "7b74aeede93a64c6a2e598f021fbaa42061a7e09e9f3f75f1c1254fbb4419368"}


def sha(a):
    return hashlib.sha256(np.ascontiguousarray(a, dtype=np.float32).tobytes()).hexdigest()


def main(out):
    os.makedirs(out, exist_ok=True)
    files = [os.path.join(out, f) for f in ("train_1M.npy", "val.npy", "test.npy")]
    if all(os.path.exists(f) for f in files):        # already prepared: verify instead of downloading
        train, val, test = (np.load(f, mmap_mode="r") for f in files)
        sample = np.linspace(0, N - 1, 1000).astype(np.int64)
        got = {"val": sha(val), "test": sha(test), "train_rows": sha(np.array(train[sample]))}
        if all(got[key] == want for key, want in FINGERPRINTS.items()):
            print("DEEP* 1M already in %s; fingerprints match the paper's" % out)
            return
    raw = os.path.join(out, "deep1b_prefix.fbin")
    nbytes = 8 + (N + EVAL) * D * 4
    if not os.path.exists(raw) or os.path.getsize(raw) != nbytes:
        req = urllib.request.Request(URL, headers={"Range": "bytes=0-%d" % (nbytes - 1)})
        with urllib.request.urlopen(req, timeout=600) as r, open(raw, "wb") as f:
            while True:
                b = r.read(1 << 24)
                if not b:
                    break
                f.write(b)
    m = np.memmap(raw, dtype=np.float32, mode="r", offset=8, shape=(N + EVAL, D))
    train, ev = m[:N], np.array(m[N:], dtype=np.float32)
    perm = np.random.RandomState(1234).permutation(EVAL)
    val, test = ev[perm[:EVAL // 2]], ev[perm[EVAL // 2:]]
    sample = np.linspace(0, N - 1, 1000).astype(np.int64)
    got = {"val": sha(val), "test": sha(test), "train_rows": sha(np.array(train[sample]))}
    for key, want in FINGERPRINTS.items():
        assert got[key] == want, "%s rows differ from the paper's" % key
    np.save(os.path.join(out, "train_1M.npy"), np.array(train, dtype=np.float32))
    np.save(os.path.join(out, "val.npy"), val)
    np.save(os.path.join(out, "test.npy"), test)
    os.remove(raw)
    print("DEEP* 1M ready in %s; fingerprints match the paper's" % out)


if __name__ == "__main__":
    main(sys.argv[1])