Download benchmarks/get_paper1m.py from Nabidnur/PISCES: direct link, hf CLI and curl.
- Browser
- Download file 4 kB
-
https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/get_paper1m.py
- Command line
-
hf download hf://Nabidnur/PISCES/benchmarks/get_paper1m.py
-
curl -L -o get_paper1m.py https://huggingface.co/Nabidnur/PISCES/resolve/main/benchmarks/get_paper1m.py
4 kB
| """Download SimSearchNet++ 1M or SPACEV 1M exactly as the e-means paper uses them, and verify. | |
| Train: the first 1,000,000 rows of the corpus. Evaluation: the 20,000 held-out rows listed in | |
| faiss benchs/e_means/paper_splits/<name>_holdout_rows.npy (copied here; see THIRD_PARTY.md), | |
| read in that order, split by permutation seed 1234 into validation (first half) and test. Each | |
| part is checked against the paper's SHA-256 fingerprints. Only the byte ranges needed are fetched | |
| (about 256 MB for SimSearchNet++, 100 MB for SPACEV). | |
| Usage: python benchmarks/get_paper1m.py {fb-ssnpp-1M|spacev-1M} DATA_DIR | |
| """ | |
| import hashlib, os, sys, urllib.request | |
| import numpy as np | |
| HERE = os.path.dirname(os.path.abspath(__file__)) | |
| N = 1_000_000 | |
| SPACEV_PART = 4294967295 | |
| SPECS = { | |
| "fb-ssnpp-1M": dict( | |
| url="https://dl.fbaipublicfiles.com/billion-scale-ann-benchmarks/FB_ssnpp_database.u8bin", | |
| d=256, dtype=np.uint8, held="fb_ssnpp_holdout_rows.npy", | |
| sha=dict(val="d51b5e50430b61a8a4a314a5df903195005da36e4de56e0a11ad3908b0a1dd08", | |
| test="fae0b470fc0a3c74bcc04245b821bf34045a199185b3ecae2c21f1c2a7b10e40", | |
| train_rows="cb4988c357268c46c8a33a55cebb5d88199badeb32a72fb6187e93aa2674274b")), | |
| "spacev-1M": dict( | |
| parts="https://huggingface.co/datasets/jkhe/spacev1b/resolve/main/vectors.bin/vectors_%d.bin", | |
| d=100, dtype=np.int8, held="spacev_holdout_rows.npy", | |
| sha=dict(val="1c59da848fc792157f107733da4a2f288568c51ea4fe44cdbbc1b9cd1352acec", | |
| test="ecb6c6d304eb772d0891685bb2cf0b417efe15924894879af7b7fcfd431a0549", | |
| train_rows="07c827691fb18f648271d20258f7e65df5bd09ffbfb5956523e7c543ab2cbd8a")), | |
| } | |
| def get(spec, a, b): | |
| """Bytes [a, b) of the (possibly multi-part) corpus file.""" | |
| out = bytearray() | |
| while a < b: | |
| if "parts" in spec: | |
| i, off = divmod(a, SPACEV_PART) | |
| stop = min(b, (i + 1) * SPACEV_PART) | |
| url, start, end = spec["parts"] % (i + 1), off, off + stop - a | |
| else: | |
| url, start, end, stop = spec["url"], a, b, b | |
| req = urllib.request.Request(url, headers={"Range": "bytes=%d-%d" % (start, end - 1)}) | |
| for attempt in range(6): | |
| try: | |
| with urllib.request.urlopen(req, timeout=600) as r: | |
| chunk = r.read() | |
| break | |
| except Exception: | |
| if attempt == 5: | |
| raise | |
| assert len(chunk) == end - start, "short read" | |
| out += chunk | |
| a = stop | |
| return bytes(out) | |
| def sha(a): | |
| return hashlib.sha256(np.ascontiguousarray(a, dtype=np.float32).tobytes()).hexdigest() | |
| def main(name, out): | |
| spec = SPECS[name] | |
| d, dt = spec["d"], np.dtype(spec["dtype"]) | |
| row = d * dt.itemsize | |
| os.makedirs(out, exist_ok=True) | |
| train = np.frombuffer(get(spec, 8, 8 + N * row), dtype=dt).reshape(N, d).astype(np.float32) | |
| index = np.load(os.path.join(HERE, spec["held"])) | |
| srt = np.sort(index) | |
| runs = np.split(srt, np.flatnonzero(np.diff(srt) != 1) + 1) | |
| rows = {} | |
| for r in runs: | |
| blob = np.frombuffer(get(spec, 8 + int(r[0]) * row, 8 + (int(r[-1]) + 1) * row), dtype=dt) | |
| for j, v in zip(r, blob.reshape(len(r), d)): | |
| rows[int(j)] = v | |
| ev = np.stack([rows[int(j)] for j in index]).astype(np.float32) | |
| perm = np.random.RandomState(1234).permutation(len(ev)) | |
| half = int(round(0.5 * len(ev))) | |
| val, test = ev[perm[:half]], ev[perm[half:]] | |
| sample = np.linspace(0, N - 1, 1000).astype(np.int64) | |
| got = dict(val=sha(val), test=sha(test), train_rows=sha(train[sample])) | |
| for key, want in spec["sha"].items(): | |
| assert got[key] == want, "%s: %s rows differ from the paper's" % (name, key) | |
| np.save(os.path.join(out, "train_1M.npy"), train) | |
| np.save(os.path.join(out, "test.npy"), test) | |
| print("%s ready in %s; fingerprints match the paper's" % (name, out)) | |
| if __name__ == "__main__": | |
| main(sys.argv[1], sys.argv[2]) | |