PISCES / benchmarks /get_paper1m.py
Nabidnur's picture
PISCES (ARIES #1): code, paper, notebooks, results and figures
c09a3f4 verified
Raw History Blame Contribute Delete
4 kB
"""Download SimSearchNet++ 1M or SPACEV 1M exactly as the e-means paper uses them, and verify.
Train: the first 1,000,000 rows of the corpus. Evaluation: the 20,000 held-out rows listed in
faiss benchs/e_means/paper_splits/<name>_holdout_rows.npy (copied here; see THIRD_PARTY.md),
read in that order, split by permutation seed 1234 into validation (first half) and test. Each
part is checked against the paper's SHA-256 fingerprints. Only the byte ranges needed are fetched
(about 256 MB for SimSearchNet++, 100 MB for SPACEV).
Usage: python benchmarks/get_paper1m.py {fb-ssnpp-1M|spacev-1M} DATA_DIR
"""
import hashlib, os, sys, urllib.request
import numpy as np
HERE = os.path.dirname(os.path.abspath(__file__))
N = 1_000_000
SPACEV_PART = 4294967295
SPECS = {
"fb-ssnpp-1M": dict(
url="https://dl.fbaipublicfiles.com/billion-scale-ann-benchmarks/FB_ssnpp_database.u8bin",
d=256, dtype=np.uint8, held="fb_ssnpp_holdout_rows.npy",
sha=dict(val="d51b5e50430b61a8a4a314a5df903195005da36e4de56e0a11ad3908b0a1dd08",
test="fae0b470fc0a3c74bcc04245b821bf34045a199185b3ecae2c21f1c2a7b10e40",
train_rows="cb4988c357268c46c8a33a55cebb5d88199badeb32a72fb6187e93aa2674274b")),
"spacev-1M": dict(
parts="https://huggingface.co/datasets/jkhe/spacev1b/resolve/main/vectors.bin/vectors_%d.bin",
d=100, dtype=np.int8, held="spacev_holdout_rows.npy",
sha=dict(val="1c59da848fc792157f107733da4a2f288568c51ea4fe44cdbbc1b9cd1352acec",
test="ecb6c6d304eb772d0891685bb2cf0b417efe15924894879af7b7fcfd431a0549",
train_rows="07c827691fb18f648271d20258f7e65df5bd09ffbfb5956523e7c543ab2cbd8a")),
}
def get(spec, a, b):
"""Bytes [a, b) of the (possibly multi-part) corpus file."""
out = bytearray()
while a < b:
if "parts" in spec:
i, off = divmod(a, SPACEV_PART)
stop = min(b, (i + 1) * SPACEV_PART)
url, start, end = spec["parts"] % (i + 1), off, off + stop - a
else:
url, start, end, stop = spec["url"], a, b, b
req = urllib.request.Request(url, headers={"Range": "bytes=%d-%d" % (start, end - 1)})
for attempt in range(6):
try:
with urllib.request.urlopen(req, timeout=600) as r:
chunk = r.read()
break
except Exception:
if attempt == 5:
raise
assert len(chunk) == end - start, "short read"
out += chunk
a = stop
return bytes(out)
def sha(a):
return hashlib.sha256(np.ascontiguousarray(a, dtype=np.float32).tobytes()).hexdigest()
def main(name, out):
spec = SPECS[name]
d, dt = spec["d"], np.dtype(spec["dtype"])
row = d * dt.itemsize
os.makedirs(out, exist_ok=True)
train = np.frombuffer(get(spec, 8, 8 + N * row), dtype=dt).reshape(N, d).astype(np.float32)
index = np.load(os.path.join(HERE, spec["held"]))
srt = np.sort(index)
runs = np.split(srt, np.flatnonzero(np.diff(srt) != 1) + 1)
rows = {}
for r in runs:
blob = np.frombuffer(get(spec, 8 + int(r[0]) * row, 8 + (int(r[-1]) + 1) * row), dtype=dt)
for j, v in zip(r, blob.reshape(len(r), d)):
rows[int(j)] = v
ev = np.stack([rows[int(j)] for j in index]).astype(np.float32)
perm = np.random.RandomState(1234).permutation(len(ev))
half = int(round(0.5 * len(ev)))
val, test = ev[perm[:half]], ev[perm[half:]]
sample = np.linspace(0, N - 1, 1000).astype(np.int64)
got = dict(val=sha(val), test=sha(test), train_rows=sha(train[sample]))
for key, want in spec["sha"].items():
assert got[key] == want, "%s: %s rows differ from the paper's" % (name, key)
np.save(os.path.join(out, "train_1M.npy"), train)
np.save(os.path.join(out, "test.npy"), test)
print("%s ready in %s; fingerprints match the paper's" % (name, out))
if __name__ == "__main__":
main(sys.argv[1], sys.argv[2])