examples/compute_similarity.py

import sys
from pathlib import Path

import torch

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from scripts.omniret_embedding import OmniRetEmbedder

IDS = ("-1rZFviqTTQ_000003", "-25e5qcELvw_000011")
EXPECTED = torch.tensor(
    [
        [0.32468632, -0.03668483, 0.28203371, 0.04291208, 0.42771006, 0.06420071],
        [-0.03580582, 0.16362236, -0.03176199, 0.33771127, -0.00980220, 0.16253276],
        [0.41171762, 0.00378584, 0.22568333, 0.01310393, 0.43492085, 0.08068858],
    ]
)

queries = [
    {
        "instruction": "Retrieve the video that aligns with the audio.",
        "audio": ROOT / "examples" / "audios" / f"{IDS[0]}.wav",
    },
    {
        "instruction": "Retrieve the audio that matches the given image.",
        "image": ROOT / "examples" / "images" / f"{IDS[1]}.jpg",
    },
    {
        "instruction": "Retrieve the audio that matches the given video.",
        "video": ROOT / "examples" / "videos" / f"{IDS[0]}.mp4",
    },
]
documents = [
    {modality: ROOT / "examples" / f"{modality}s" / f"{media_id}.{extension}"}
    for modality, extension in (("audio", "wav"), ("image", "jpg"), ("video", "mp4"))
    for media_id in IDS
]

model = OmniRetEmbedder(ROOT, torch_dtype=torch.bfloat16)
embeddings = model.process(queries + documents)
scores = embeddings[: len(queries)] @ embeddings[len(queries) :].T
print(scores.tolist())
if not torch.allclose(scores.cpu(), EXPECTED, atol=1e-3, rtol=0):
    raise SystemExit("scores differ from checkpoint-2220 expectations")