"""Download SimSearchNet++ 1M or SPACEV 1M exactly as the e-means paper uses them, and verify. Train: the first 1,000,000 rows of the corpus. Evaluation: the 20,000 held-out rows listed in faiss benchs/e_means/paper_splits/_holdout_rows.npy (copied here; see THIRD_PARTY.md), read in that order, split by permutation seed 1234 into validation (first half) and test. Each part is checked against the paper's SHA-256 fingerprints. Only the byte ranges needed are fetched (about 256 MB for SimSearchNet++, 100 MB for SPACEV). Usage: python benchmarks/get_paper1m.py {fb-ssnpp-1M|spacev-1M} DATA_DIR """ import hashlib, os, sys, urllib.request import numpy as np HERE = os.path.dirname(os.path.abspath(__file__)) N = 1_000_000 SPACEV_PART = 4294967295 SPECS = { "fb-ssnpp-1M": dict( url="https://dl.fbaipublicfiles.com/billion-scale-ann-benchmarks/FB_ssnpp_database.u8bin", d=256, dtype=np.uint8, held="fb_ssnpp_holdout_rows.npy", sha=dict(val="d51b5e50430b61a8a4a314a5df903195005da36e4de56e0a11ad3908b0a1dd08", test="fae0b470fc0a3c74bcc04245b821bf34045a199185b3ecae2c21f1c2a7b10e40", train_rows="cb4988c357268c46c8a33a55cebb5d88199badeb32a72fb6187e93aa2674274b")), "spacev-1M": dict( parts="https://huggingface.co/datasets/jkhe/spacev1b/resolve/main/vectors.bin/vectors_%d.bin", d=100, dtype=np.int8, held="spacev_holdout_rows.npy", sha=dict(val="1c59da848fc792157f107733da4a2f288568c51ea4fe44cdbbc1b9cd1352acec", test="ecb6c6d304eb772d0891685bb2cf0b417efe15924894879af7b7fcfd431a0549", train_rows="07c827691fb18f648271d20258f7e65df5bd09ffbfb5956523e7c543ab2cbd8a")), } def get(spec, a, b): """Bytes [a, b) of the (possibly multi-part) corpus file.""" out = bytearray() while a < b: if "parts" in spec: i, off = divmod(a, SPACEV_PART) stop = min(b, (i + 1) * SPACEV_PART) url, start, end = spec["parts"] % (i + 1), off, off + stop - a else: url, start, end, stop = spec["url"], a, b, b req = urllib.request.Request(url, headers={"Range": "bytes=%d-%d" % (start, end - 1)}) for attempt in range(6): try: with urllib.request.urlopen(req, timeout=600) as r: chunk = r.read() break except Exception: if attempt == 5: raise assert len(chunk) == end - start, "short read" out += chunk a = stop return bytes(out) def sha(a): return hashlib.sha256(np.ascontiguousarray(a, dtype=np.float32).tobytes()).hexdigest() def main(name, out): spec = SPECS[name] d, dt = spec["d"], np.dtype(spec["dtype"]) row = d * dt.itemsize os.makedirs(out, exist_ok=True) train = np.frombuffer(get(spec, 8, 8 + N * row), dtype=dt).reshape(N, d).astype(np.float32) index = np.load(os.path.join(HERE, spec["held"])) srt = np.sort(index) runs = np.split(srt, np.flatnonzero(np.diff(srt) != 1) + 1) rows = {} for r in runs: blob = np.frombuffer(get(spec, 8 + int(r[0]) * row, 8 + (int(r[-1]) + 1) * row), dtype=dt) for j, v in zip(r, blob.reshape(len(r), d)): rows[int(j)] = v ev = np.stack([rows[int(j)] for j in index]).astype(np.float32) perm = np.random.RandomState(1234).permutation(len(ev)) half = int(round(0.5 * len(ev))) val, test = ev[perm[:half]], ev[perm[half:]] sample = np.linspace(0, N - 1, 1000).astype(np.int64) got = dict(val=sha(val), test=sha(test), train_rows=sha(train[sample])) for key, want in spec["sha"].items(): assert got[key] == want, "%s: %s rows differ from the paper's" % (name, key) np.save(os.path.join(out, "train_1M.npy"), train) np.save(os.path.join(out, "test.npy"), test) print("%s ready in %s; fingerprints match the paper's" % (name, out)) if __name__ == "__main__": main(sys.argv[1], sys.argv[2])