benchmarks/embedding_bench.py

"""Benchmark the frozen binary vocabulary on word similarity (WordSim-353).

Every token has a fixed 512-bit code. If the codes carry meaning, similar words should
have nearby codes. This script scores each human-rated word pair by Hamming similarity
between the two words' codes and reports the Spearman correlation with the human ratings.
Random codes score about 0. Runs offline on the files in this repo.

    python benchmarks/embedding_bench.py
"""
import csv, pathlib, sys
import numpy as np
HERE = pathlib.Path(__file__).resolve().parent; ROOT = HERE.parent
sys.path.insert(0, str(ROOT))
from shadow_runtime.retriever import enc

fp = np.unpackbits(np.load(ROOT / "fp131072.npy"), axis=1)[:, :512]
rows = [l.split(",") for l in open(HERE / "wordsim353.csv", encoding="utf-8").read().splitlines() if l]
rng = np.random.default_rng(0); rand = rng.integers(0, 2, size=fp.shape).astype(np.uint8)

def score(table):
    xs, ys = [], []
    for w1, w2, human in rows:
        i1, i2 = enc(" " + w1.lower()), enc(" " + w2.lower())
        if len(i1) != 1 or len(i2) != 1: continue
        xs.append(1 - np.mean(table[i1[0]] != table[i2[0]])); ys.append(float(human))
    from scipy.stats import spearmanr
    return spearmanr(xs, ys).statistic, len(xs)

r, n = score(fp); r0, _ = score(rand)
print(f"WordSim-353, single-token pairs (n={n})")
print(f"  frozen vocabulary codes : Spearman {r:.3f}")
print(f"  random codes (baseline) : Spearman {r0:.3f}")