lorelaiassistant's picture
EmbeddingGemma 2 text+vision Core ML with int8 weights
45446c9 verified
Raw History Blame Contribute Delete
1 kB
"""embeddings.bf16 (262144 x 512) -> embeddings.i8 (int8, row-major) and
embeddings_scale.f16 (one fp16 scale per row, symmetric: value = q * scale).
usage: table_int8.py <fp16 dir> <out dir>
"""
import sys
from pathlib import Path
import numpy as np
src, out = Path(sys.argv[1]), Path(sys.argv[2])
raw = np.fromfile(src / "embeddings.bf16", dtype=np.uint16).reshape(-1, 512)
table = (raw.astype(np.uint32) << 16).view(np.float32)
scale = np.abs(table).max(axis=1) / 127.0
scale16 = scale.astype(np.float16)
safe = np.where(scale16 == 0, 1, scale16).astype(np.float32)
q = np.clip(np.round(table / safe[:, None]), -127, 127).astype(np.int8)
q.tofile(out / "embeddings.i8")
scale16.tofile(out / "embeddings_scale.f16")
err = q.astype(np.float32) * scale16.astype(np.float32)[:, None] - table
rel = np.linalg.norm(err, axis=1) / np.maximum(np.linalg.norm(table, axis=1), 1e-12)
print("rows", table.shape[0], "relative row error mean", rel.mean(), "p99", np.quantile(rel, 0.99),
"max", rel.max())