Buckets:
| """Benchmark the frozen binary vocabulary on word similarity (WordSim-353). | |
| Every token has a fixed 512-bit code. If the codes carry meaning, similar words should | |
| have nearby codes. This script scores each human-rated word pair by Hamming similarity | |
| between the two words' codes and reports the Spearman correlation with the human ratings. | |
| Random codes score about 0. Runs offline on the files in this repo. | |
| python benchmarks/embedding_bench.py | |
| """ | |
| import csv, pathlib, sys | |
| import numpy as np | |
| HERE = pathlib.Path(__file__).resolve().parent; ROOT = HERE.parent | |
| sys.path.insert(0, str(ROOT)) | |
| from shadow_runtime.retriever import enc | |
| fp = np.unpackbits(np.load(ROOT / "fp131072.npy"), axis=1)[:, :512] | |
| rows = [l.split(",") for l in open(HERE / "wordsim353.csv", encoding="utf-8").read().splitlines() if l] | |
| rng = np.random.default_rng(0); rand = rng.integers(0, 2, size=fp.shape).astype(np.uint8) | |
| def score(table): | |
| xs, ys = [], [] | |
| for w1, w2, human in rows: | |
| i1, i2 = enc(" " + w1.lower()), enc(" " + w2.lower()) | |
| if len(i1) != 1 or len(i2) != 1: continue | |
| xs.append(1 - np.mean(table[i1[0]] != table[i2[0]])); ys.append(float(human)) | |
| from scipy.stats import spearmanr | |
| return spearmanr(xs, ys).statistic, len(xs) | |
| r, n = score(fp); r0, _ = score(rand) | |
| print(f"WordSim-353, single-token pairs (n={n})") | |
| print(f" frozen vocabulary codes : Spearman {r:.3f}") | |
| print(f" random codes (baseline) : Spearman {r0:.3f}") | |
Xet Storage Details
- Size:
- 1.45 kB
- Xet hash:
- 5596d947ce5387250ab7c8734a762cef50984b6f1b57d22edd857617a0f76166
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.