| |
| """MTEB(por, v2) random baseline encoder — the FLOOR reference. |
| |
| Design: each text -> a deterministic L2-normalized random vector, seeded by |
| sha256(SEED|text). Deterministic-per-text => reproducible + zero semantic signal |
| (textually-different but semantically-similar sentences get unrelated vectors). |
| Gives chance-level performance across STS / retrieval / classification / clustering. |
| No GPU. Runs locally with the same pinned-SHA mteb_pt tasks as the real models. |
| """ |
| import os |
| import hashlib |
| import numpy as np |
| import mteb |
| import mteb_pt |
| import mteb_pt.register as register |
| from mteb.models.abs_encoder import AbsEncoder |
| from mteb.models.model_meta import ModelMeta |
|
|
| DIM = 768 |
| SEED = 42 |
| _EXCLUDED = {"OffComBR", "CSTNewsClustering", "BBCNewsPTClustering", "TweetSentBR"} |
|
|
|
|
| class RandomEncoder(AbsEncoder): |
| def __init__(self): |
| self.mteb_model_meta = ModelMeta( |
| loader=None, |
| name="mteb-pt/baseline-random-encoder", |
| revision="1.0", |
| release_date="2026-06-28", |
| languages=["por-Latn"], |
| n_parameters=0, |
| memory_usage_mb=None, |
| max_tokens=None, |
| embed_dim=DIM, |
| license=None, |
| open_weights=True, |
| public_training_code=None, |
| public_training_data=None, |
| framework=[], |
| similarity_fn_name="cosine", |
| use_instructions=False, |
| training_datasets=None, |
| ) |
|
|
| def encode(self, inputs, *, task_metadata, hf_split, hf_subset, prompt_type=None, **kwargs): |
| texts = [t for batch in inputs for t in batch["text"]] |
| out = np.empty((len(texts), DIM), dtype=np.float32) |
| for i, t in enumerate(texts): |
| h = int(hashlib.sha256((str(SEED) + "|" + (t or "")).encode("utf-8")).hexdigest(), 16) % (2**32) |
| v = np.random.default_rng(h).standard_normal(DIM).astype(np.float32) |
| out[i] = v / (np.linalg.norm(v) + 1e-9) |
| return out |
|
|
|
|
| def load_tasks(only=None): |
| tasks = [cls() for cls in register._TASKS_TO_REGISTER if cls.metadata.name not in _EXCLUDED] |
| tasks.append(mteb.get_task("Assin2STS")) |
| if only: |
| keep = {x.strip() for x in only.split(",")} |
| tasks = [t for t in tasks if t.metadata.name in keep] |
| return tasks |
|
|
|
|
| if __name__ == "__main__": |
| tasks = load_tasks(os.environ.get("RB_TASKS")) |
| print(f"[random-baseline] {len(tasks)} tasks: {sorted(t.metadata.name for t in tasks)}", flush=True) |
| mteb.evaluate(RandomEncoder(), tasks=tasks, overwrite_strategy="always", raise_error=False, show_progress_bar=False) |
| print("[random-baseline] DONE", flush=True) |
|
|
