shawkat's picture
papr-embed-code-1-0.6b for the Apple Neural Engine: multifunction fp16 (len 64/256/1024), 17 ms/query on M2 Pro, cos 0.9998
1bb8146 verified
Raw History Blame Contribute Delete
2.29 kB
"""papr-embed-code-1-0.6b on the Apple Neural Engine. Apple Silicon, macOS 15+.
pip install coremltools tokenizers huggingface_hub numpy
python examples/embed.py
"""
import numpy as np
import coremltools as ct
from huggingface_hub import snapshot_download
from tokenizers import Tokenizer
REPO = "papr-ai/papr-embed-code-1-0.6b-CoreML"
EOS = 151643 # <|endoftext|>. NOT tokenizer.eos_token (<|im_end|>, 151645): that silently gives wrong vectors
LENGTHS = (64, 256, 1024) # one compiled function per fixed length
INSTRUCT = "Given a code search query about a repository, retrieve the source code file that must be read or edited"
# local_dir matters: CoreML cannot compile through the symlinks in the default HF cache ("weight.bin doesn't exist")
d = snapshot_download(REPO, local_dir="papr-embed-code-1-0.6b-CoreML")
tok = Tokenizer.from_file(f"{d}/tokenizer.json")
_models = {} # first load per length compiles for the Neural Engine (~30-60 s); macOS caches it
def _model(L):
if L not in _models:
_models[L] = ct.models.MLModel(f"{d}/papr-embed-code-1-0.6b.mlpackage",
compute_units=ct.ComputeUnit.CPU_AND_NE, function_name=f"len_{L}")
return _models[L]
def embed(text: str) -> np.ndarray:
"""1024-d unit vector. Text longer than 1023 tokens is truncated: use the ONNX/PyTorch build for long files."""
ids = tok.encode(text, add_special_tokens=False).ids[: LENGTHS[-1] - 1] + [EOS]
L = next(l for l in LENGTHS if l >= len(ids))
x = np.zeros((1, L), np.int32)
x[0, : len(ids)] = ids
out = _model(L).predict({"input_ids": x, "last_index": np.array([len(ids) - 1], np.int32)})
return out["embedding"][0].astype(np.float32)
def embed_query(q: str) -> np.ndarray:
return embed(f"Instruct: {INSTRUCT}\nQuery:{q}") # queries get the instruction; documents never do
if __name__ == "__main__":
q = embed_query("retry an upload with exponential backoff")
docs = ["async function upload(f){ for (let i=0;i<5;i++){ try { return await send(f) } catch { await sleep(2**i*200) } } }",
"SELECT name, email FROM users WHERE created_at > now() - interval '7 days';"]
print([round(float(q @ embed(t)), 4) for t in docs]) # the retry snippet should score clearly higher