#!/usr/bin/env python """MTEB(por, v2) random baseline encoder — the FLOOR reference. Design: each text -> a deterministic L2-normalized random vector, seeded by sha256(SEED|text). Deterministic-per-text => reproducible + zero semantic signal (textually-different but semantically-similar sentences get unrelated vectors). Gives chance-level performance across STS / retrieval / classification / clustering. No GPU. Runs locally with the same pinned-SHA mteb_pt tasks as the real models. """ import os import hashlib import numpy as np import mteb import mteb_pt # noqa: F401 (registers task modules) import mteb_pt.register as register from mteb.models.abs_encoder import AbsEncoder from mteb.models.model_meta import ModelMeta DIM = 768 SEED = 42 _EXCLUDED = {"OffComBR", "CSTNewsClustering", "BBCNewsPTClustering", "TweetSentBR"} class RandomEncoder(AbsEncoder): def __init__(self): self.mteb_model_meta = ModelMeta( loader=None, name="mteb-pt/baseline-random-encoder", revision="1.0", release_date="2026-06-28", languages=["por-Latn"], n_parameters=0, memory_usage_mb=None, max_tokens=None, embed_dim=DIM, license=None, open_weights=True, public_training_code=None, public_training_data=None, framework=[], similarity_fn_name="cosine", use_instructions=False, training_datasets=None, ) def encode(self, inputs, *, task_metadata, hf_split, hf_subset, prompt_type=None, **kwargs): texts = [t for batch in inputs for t in batch["text"]] out = np.empty((len(texts), DIM), dtype=np.float32) for i, t in enumerate(texts): h = int(hashlib.sha256((str(SEED) + "|" + (t or "")).encode("utf-8")).hexdigest(), 16) % (2**32) v = np.random.default_rng(h).standard_normal(DIM).astype(np.float32) out[i] = v / (np.linalg.norm(v) + 1e-9) return out def load_tasks(only=None): tasks = [cls() for cls in register._TASKS_TO_REGISTER if cls.metadata.name not in _EXCLUDED] tasks.append(mteb.get_task("Assin2STS")) if only: keep = {x.strip() for x in only.split(",")} tasks = [t for t in tasks if t.metadata.name in keep] return tasks if __name__ == "__main__": tasks = load_tasks(os.environ.get("RB_TASKS")) print(f"[random-baseline] {len(tasks)} tasks: {sorted(t.metadata.name for t in tasks)}", flush=True) mteb.evaluate(RandomEncoder(), tasks=tasks, overwrite_strategy="always", raise_error=False, show_progress_bar=False) print("[random-baseline] DONE", flush=True)