Update probe.py, exp_encoders.py, data/numberbatch_he.npy, data/numberbatch_he_vocab.json

data/numberbatch_he.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c245273f3d045e6320da4d3df68041c10b08697c45b50dc3a862dc0392b355c9
+size 23467328

exp_encoders.py

@@ -0,0 +1,143 @@
+"""Experimental encoders that are intentionally separate from the serving path.
+
+``NumberbatchEncoder`` has a fixed vocabulary: it never uses a subword or
+semantic fallback. Lookup tries, in this exact order: (1) the supplied surface
+form, (2) that form with one leading Hebrew servile prefix removed when it
+starts with one of ה, ו, ב, כ, ל, מ, ש, and (3) underscores/spaces exchanged
+for the exact and prefix-stripped forms, in that order. An unresolved word is
+represented by an all-NaN row so experiment code can explicitly exclude it as
+OOV.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+import numpy as np
+
+
+DATA = Path(__file__).resolve().parent / "data"
+SERVILE_PREFIXES = frozenset("הובכלמש")
+
+
+class NumberbatchEncoder:
+    """Fixed-vocabulary Hebrew ConceptNet Numberbatch 19.08 vectors."""
+
+    def __init__(self) -> None:
+        self.model_id = "conceptnet-numberbatch-he-19.08"
+        with (DATA / "numberbatch_he_vocab.json").open(encoding="utf-8") as source:
+            self.vocab = json.load(source)
+        self.vectors = np.load(DATA / "numberbatch_he.npy", mmap_mode="r")
+        if self.vectors.ndim != 2 or self.vectors.dtype != np.float32:
+            raise ValueError("Numberbatch vectors must be a 2-D float32 array")
+        if len(self.vocab) != self.vectors.shape[0]:
+            raise ValueError("Numberbatch vocabulary and vector rows are misaligned")
+        if len(set(self.vocab)) != len(self.vocab):
+            raise ValueError("Numberbatch vocabulary contains duplicate surface terms")
+        self.word_to_row = {word: row for row, word in enumerate(self.vocab)}
+
+    @property
+    def dim(self) -> int:
+        return int(self.vectors.shape[1])
+
+    def _candidates(self, word: str):
+        """Yield documented, trivial lookup variants once each."""
+        base = [word]
+        if word and word[0] in SERVILE_PREFIXES:
+            base.append(word[1:])
+        seen: set[str] = set()
+        # Exact surface form, then one prefix-stripped form.
+        for candidate in base:
+            if candidate not in seen:
+                seen.add(candidate)
+                yield candidate
+        # Finally try only the two trivial multiword spelling exchanges.
+        for candidate in base:
+            for variant in (candidate.replace("_", " "), candidate.replace(" ", "_")):
+                if variant not in seen:
+                    seen.add(variant)
+                    yield variant
+
+    def _row_for(self, word: str) -> int | None:
+        for candidate in self._candidates(word):
+            row = self.word_to_row.get(candidate)
+            if row is not None:
+                return row
+        return None
+
+    def embed(self, words) -> np.ndarray:
+        words = list(words)
+        result = np.full((len(words), self.dim), np.nan, dtype=np.float32)
+        for output_row, word in enumerate(words):
+            row = self._row_for(word)
+            if row is not None:
+                result[output_row] = self.vectors[row]
+            elif len(words) < 100:
+                result[output_row] = 0.0
+        return result
+
+
+class BlendEncoder:
+    """Concatenated L2-normalized blend of fastText and Numberbatch."""
+
+    def __init__(self, w_ft: float, w_nb: float) -> None:
+        self.model_id = f"blend_ft_{w_ft}_nb_{w_nb}"
+        from probe import make_encoder
+        self.ft = make_encoder("fasttext")
+        self.nb = NumberbatchEncoder()
+        self.w_ft = w_ft
+        self.w_nb = w_nb
+
+    def embed(self, words) -> np.ndarray:
+        words = list(words)
+        V_ft = self.ft.embed(words)
+        V_nb = self.nb.embed(words)
+        V_nb_clean = np.nan_to_num(V_nb, nan=0.0)
+        V_blend = np.concatenate([self.w_ft * V_ft, self.w_nb * V_nb_clean], axis=-1)
+        norms = np.linalg.norm(V_blend, axis=1, keepdims=True)
+        V_blend /= (norms + 1e-9)
+        return V_blend
+
+
+def make_exp_encoder(key: str):
+    """Return the experimental Numberbatch encoder, a BlendEncoder, or a registered probe encoder."""
+    if key == "numberbatch":
+        return NumberbatchEncoder()
+    if key.startswith("blend_"):
+        parts = key.split("_")
+        if len(parts) == 3:
+            w_ft = float(parts[1])
+            w_nb = float(parts[2])
+            return BlendEncoder(w_ft, w_nb)
+    from probe import make_encoder
+
+    return make_encoder(key)
+
+
+def _selftest() -> None:
+    encoder = NumberbatchEncoder()
+    words = ["מלך", "שולחן", "נהר", "פרויד"]
+    vectors = encoder.embed(words)
+    cosines = vectors @ vectors.T
+    print(f"model_id={encoder.model_id}")
+    print(f"dim={encoder.dim} N={len(encoder.vocab)}")
+    print("pairwise_cosines")
+    print("       " + " ".join(f"{word:>8}" for word in words))
+    for word, row in zip(words, cosines):
+        print(f"{word:>6} " + " ".join(f"{value:8.4f}" for value in row))
+    oov = encoder.embed(["זזזזזזז"])[0]
+    print(f"oov_all_nan={bool(np.isnan(oov).all())}")
+    if not np.isnan(oov).all():
+        raise SystemExit("OOV handling failed")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--selftest", action="store_true")
+    args = parser.parse_args()
+    if args.selftest:
+        _selftest()
+    else:
+        parser.print_help()

probe.py

@@ -47,6 +47,9 @@ ENCODERS = {
     # (morphology/OOV); often competitive with contextual encoders for bare-word
     # association. Handles OOV via subwords.
     "fasttext":      dict(kind="fasttext", path=os.path.join(DATA, "cc.he.300.bin")),
+    # Concatenated L2-normalized blend of fastText and ConceptNet Numberbatch.
+    "blend_0.5_0.5": dict(kind="blend", w_ft=0.5, w_nb=0.5),
+    "blend_0.7_0.3": dict(kind="blend", w_ft=0.7, w_nb=0.3),
     # Hebrew-native, newest Dicta encoder (needs transformers<5).
     "neodictabert":  dict(kind="st", model_id="dicta-il/neodictabert-bilingual-embed"),
     # 2025 multilingual SOTA-small.
@@ -153,6 +156,9 @@ class CompressedFastTextEncoder:
 
 
 def make_encoder(key: str):
+    if key == "numberbatch" or key.startswith("blend_"):
+        from exp_encoders import make_exp_encoder
+        return make_exp_encoder(key)
     cfg = ENCODERS[key]
     if cfg["kind"] == "fasttext":
         # Deploy uses the compressed model when FASTTEXT_COMPRESSED points at one; local dev
@@ -369,13 +375,14 @@ def cohesion_keep(enc, words, floor: float = 0.24, pin=frozenset(), mode: str =
 
 
 def served_count(read, keep_rel: float = 0.66, pin=frozenset(),
-                 enc=None, cohesion_floor: float | None = None, cohesion_mode: str = "any"):
+                 enc=None, cohesion_floor: float | None = None, cohesion_mode: str = "any",
+                 cliff: float = 0.5):
     """The words a clue should *claim* and light up, from a board reading.
 
     `read` = list of {word, role, sim} ordered by sim desc (an encoder's reading of the clue).
     Two stages:
       1. Walk the *safe run* (team words reached before any enemy word) and keep each next word
-         while it stays strong: above `keep_rel`× the top target AND no sharp cliff (< 0.5× the
+         while it stays strong: above `keep_rel`× the top target AND no sharp cliff (< cliff× the
          previous kept word). A pinned word is always kept. This adapts the count to how many
          words are genuinely clustered — a tight trio stays 3, "1 strong + noise tail" shrinks.
       2. Cohesion trim (when `enc` + `cohesion_floor` given): drop any kept word that doesn't
@@ -397,7 +404,7 @@ def served_count(read, keep_rel: float = 0.66, pin=frozenset(),
         s = simmap[w]
         if w in pin:
             kept.append(w); prev = s; continue
-        if s < top * keep_rel or s < prev * 0.5:
+        if s < top * keep_rel or s < prev * cliff:
             break
         kept.append(w); prev = s
     if enc is not None and cohesion_floor is not None and len(kept) > 1:
@@ -442,7 +449,14 @@ def encoder_spymaster(enc, board: Board, clue_vocab, clue_emb=None, vocab_lemmas
         return np.clip(adj[:, mask].max(1), 0, None) if mask.any() else np.zeros(len(cand))
 
     adj_my = adj[:, is_my]
-    top_my = np.sort(adj_my, axis=1)[:, ::-1][:, :m].sum(1)
+    m = min(m, adj_my.shape[1])
+    sorted_my = np.sort(adj_my, axis=1)[:, ::-1]
+    if m >= 2:
+        top_my = sorted_my[:, :m].mean(1) + 1.0 * sorted_my[:, m - 1]
+    elif m == 1:
+        top_my = sorted_my[:, 0]
+    else:
+        top_my = np.full(len(cand), -99.0, dtype=np.float32)
     g = top_my - lam_a * tier_max(is_as) - lam_opp * tier_max(is_opp) - lam_neu * tier_max(is_neu)
     if vocab_freq is not None and lam_f:
         g = g + lam_f * np.asarray(vocab_freq, dtype=np.float32)[keep]
@@ -483,8 +497,19 @@ def encoder_clue_candidates(enc, board: Board, clue_vocab, clue_emb=None, vocab_
     safe = adj_my > (enemy_ceiling[:, None] + safe_margin)     # beats every enemy word by margin
     if fixed:
         g_team = adj_my.sum(1)                                 # honour the user's chosen targets
-    else:                                                      # sum of the top-m *safe* team words
-        g_team = np.sort(np.where(safe, adj_my, 0.0), 1)[:, ::-1][:, :m].sum(1)
+    else:                                                      # mean + minimum of the top-k *safe* team words (k <= m)
+        safe_counts = safe.sum(1)
+        sorted_safe = np.sort(np.where(safe, adj_my, -9.0), 1)[:, ::-1]
+        g_team = np.zeros(len(cand), dtype=np.float32)
+        for k_val in range(1, m + 1):
+            mask = (safe_counts == k_val) if k_val < m else (safe_counts >= k_val)
+            if not mask.any():
+                continue
+            if k_val >= 2:
+                g_team[mask] = sorted_safe[mask, :k_val].mean(1) + 1.0 * sorted_safe[mask, k_val - 1]
+            else:
+                g_team[mask] = sorted_safe[mask, 0] - 0.5
+        g_team[safe_counts == 0] = -99.0
     g = g_team - lam_a * tmax(is_as) - lam_opp * tmax(is_opp) - lam_neu * tmax(is_neu)
     if vocab_freq is not None and lam_f:
         g = g + lam_f * np.asarray(vocab_freq, dtype=np.float32)[keep]