add reproducible eval script

run_random_baseline.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+"""MTEB(por, v2) random baseline encoder — the FLOOR reference.
+
+Design: each text -> a deterministic L2-normalized random vector, seeded by
+sha256(SEED|text). Deterministic-per-text => reproducible + zero semantic signal
+(textually-different but semantically-similar sentences get unrelated vectors).
+Gives chance-level performance across STS / retrieval / classification / clustering.
+No GPU. Runs locally with the same pinned-SHA mteb_pt tasks as the real models.
+"""
+import os
+import hashlib
+import numpy as np
+import mteb
+import mteb_pt  # noqa: F401 (registers task modules)
+import mteb_pt.register as register
+from mteb.models.abs_encoder import AbsEncoder
+from mteb.models.model_meta import ModelMeta
+
+DIM = 768
+SEED = 42
+_EXCLUDED = {"OffComBR", "CSTNewsClustering", "BBCNewsPTClustering", "TweetSentBR"}
+
+
+class RandomEncoder(AbsEncoder):
+    def __init__(self):
+        self.mteb_model_meta = ModelMeta(
+            loader=None,
+            name="mteb-pt/baseline-random-encoder",
+            revision="1.0",
+            release_date="2026-06-28",
+            languages=["por-Latn"],
+            n_parameters=0,
+            memory_usage_mb=None,
+            max_tokens=None,
+            embed_dim=DIM,
+            license=None,
+            open_weights=True,
+            public_training_code=None,
+            public_training_data=None,
+            framework=[],
+            similarity_fn_name="cosine",
+            use_instructions=False,
+            training_datasets=None,
+        )
+
+    def encode(self, inputs, *, task_metadata, hf_split, hf_subset, prompt_type=None, **kwargs):
+        texts = [t for batch in inputs for t in batch["text"]]
+        out = np.empty((len(texts), DIM), dtype=np.float32)
+        for i, t in enumerate(texts):
+            h = int(hashlib.sha256((str(SEED) + "|" + (t or "")).encode("utf-8")).hexdigest(), 16) % (2**32)
+            v = np.random.default_rng(h).standard_normal(DIM).astype(np.float32)
+            out[i] = v / (np.linalg.norm(v) + 1e-9)
+        return out
+
+
+def load_tasks(only=None):
+    tasks = [cls() for cls in register._TASKS_TO_REGISTER if cls.metadata.name not in _EXCLUDED]
+    tasks.append(mteb.get_task("Assin2STS"))
+    if only:
+        keep = {x.strip() for x in only.split(",")}
+        tasks = [t for t in tasks if t.metadata.name in keep]
+    return tasks
+
+
+if __name__ == "__main__":
+    tasks = load_tasks(os.environ.get("RB_TASKS"))
+    print(f"[random-baseline] {len(tasks)} tasks: {sorted(t.metadata.name for t in tasks)}", flush=True)
+    mteb.evaluate(RandomEncoder(), tasks=tasks, overwrite_strategy="always", raise_error=False, show_progress_bar=False)
+    print("[random-baseline] DONE", flush=True)