new upload restrictions

backend/app/ccr.py

@@ -42,16 +42,39 @@ class SentenceTransformerBackend:
 
     _cache: dict[str, object] = {}
 
-    def __init__(self, model_name: str, revision: str | None = None):
+    def __init__(
+        self,
+        model_name: str,
+        revision: str | None = None,
+        pooling_fallback: str | None = None,
+        max_seq_length: int | None = None,
+    ):
         self.name = model_name
         self.revision = revision
+        self.pooling_fallback = pooling_fallback
+        self._max_seq_length = max_seq_length
 
     def _model(self):
         if self.name not in self._cache:
             from sentence_transformers import SentenceTransformer  # lazy: heavy import
 
-            kwargs = {"revision": self.revision} if self.revision else {}
-            self._cache[self.name] = SentenceTransformer(self.name, **kwargs)
+            if self.pooling_fallback:
+                # Registry-flagged repos ship modules.json without the pooling
+                # config it references, so auto-loading fails; build the module
+                # stack explicitly with the pooling the model card documents.
+                from sentence_transformers import models as st_models
+
+                margs = {"model_args": {"revision": self.revision}} if self.revision else {}
+                word = st_models.Transformer(
+                    self.name, max_seq_length=self._max_seq_length, **margs
+                )
+                # renamed in newer sentence-transformers; support both
+                get_dim = getattr(word, "get_embedding_dimension", None) or word.get_word_embedding_dimension
+                pool = st_models.Pooling(get_dim(), pooling_mode=self.pooling_fallback)
+                self._cache[self.name] = SentenceTransformer(modules=[word, pool])
+            else:
+                kwargs = {"revision": self.revision} if self.revision else {}
+                self._cache[self.name] = SentenceTransformer(self.name, **kwargs)
         return self._cache[self.name]
 
     @property
@@ -68,7 +91,7 @@ class SentenceTransformerBackend:
             emb = model.encode(
                 texts[i : i + batch],
                 convert_to_numpy=True,
-                normalize_embeddings=True,
+                normalize_embeddings=True, # scales every vector to length 1
                 show_progress_bar=False,
             )
             out.append(emb)
@@ -115,7 +138,12 @@ def get_backend(model_id: str) -> EmbeddingBackend:
     from . import registry  # local import: engine stays importable without yaml deps
 
     cfg = registry.get_model(model_id)
-    return SentenceTransformerBackend(cfg.provider_model_id, revision=cfg.pinned_revision)
+    return SentenceTransformerBackend(
+        cfg.provider_model_id,
+        revision=cfg.pinned_revision,
+        pooling_fallback=cfg.pooling_fallback,
+        max_seq_length=cfg.max_seq_length,
+    )
 
 
 @dataclass
@@ -201,8 +229,8 @@ def run_ccr(
         doc_emb = encode_unique(backend, texts_for_encoding, progress_cb=doc_progress)
 
     # Both matrices are L2-normalized -> cosine similarity is a dot product.
-    sims = doc_emb @ item_emb.T
-    scores = sims.mean(axis=1)
+    sims = doc_emb @ item_emb.T   # similarity of every text to every item
+    scores = sims.mean(axis=1)   # average across items = the CCR score
 
     finished = datetime.now(timezone.utc)
     items_hash = hashlib.sha256("\n".join(items).encode()).hexdigest()[:16]

backend/app/jobs.py

@@ -304,6 +304,8 @@ def run_job(job_id: str) -> None:
             "model_registry_id": model_cfg.id if model_cfg else job.model_name,
             "provider_model_id": model_cfg.provider_model_id if model_cfg else job.model_name,
             "model_revision": model_cfg.revision if model_cfg else None,
+            "model_pooling_fallback": model_cfg.pooling_fallback if model_cfg else None,
+            "model_max_seq_length": model_cfg.max_seq_length if model_cfg else None,
             "scoring": {"adjustment_strategy": "none", "aggregate": "mean_all_items"},
             "output_schema": (
                 list(work_df.columns)

backend/app/registry.py

@@ -49,6 +49,11 @@ class ModelConfig:
     text_prefix: str
     normalize_embeddings: bool
     lazy_load: bool
+    # Set ("mean"/"cls") for repos whose sentence-transformers packaging is
+    # incomplete (modules.json references a pooling config the repo lacks):
+    # the backend then builds Transformer+Pooling explicitly instead of
+    # relying on SentenceTransformer(id) auto-loading.
+    pooling_fallback: str | None = None
     user_warnings: tuple[str, ...] = field(default_factory=tuple)
 
     @property
@@ -89,6 +94,7 @@ def _parse_model(raw: dict, lang_sets) -> ModelConfig:
         text_prefix=usage.get("text_prefix") or sym or "",
         normalize_embeddings=bool(usage.get("normalize_embeddings", True)),
         lazy_load=bool(ops.get("lazy_load", False)),
+        pooling_fallback=usage.get("pooling_fallback") or None,
         user_warnings=tuple(raw.get("warnings", []) or []),
     )
 

backend/app/reproducibility.py

@@ -47,6 +47,23 @@ def script_text(metadata: dict) -> str:
     provider = metadata.get("provider_model_id", metadata.get("model", ""))
     revision = metadata.get("model_revision")
     revision_arg = f", revision={revision!r}" if revision and revision != "PIN_ME" else ""
+    pooling_fallback = metadata.get("model_pooling_fallback")
+    max_seq = metadata.get("model_max_seq_length")
+    if pooling_fallback:
+        # Repos with incomplete sentence-transformers packaging (pooling config
+        # missing) must be assembled module-by-module, exactly as the platform
+        # backend does - otherwise the exported script cannot load the model.
+        rev_model_args = f", model_args={{'revision': {revision!r}}}" if revision and revision != "PIN_ME" else ""
+        model_loader = (
+            f"word = st_models.Transformer({provider!r}, max_seq_length={max_seq!r}{rev_model_args})\n"
+            f"    get_dim = getattr(word, 'get_embedding_dimension', None) or word.get_word_embedding_dimension\n"
+            f"    pool = st_models.Pooling(get_dim(), pooling_mode={pooling_fallback!r})\n"
+            f"    model = SentenceTransformer(modules=[word, pool])"
+        )
+        st_import = "from sentence_transformers import SentenceTransformer\nfrom sentence_transformers import models as st_models"
+    else:
+        model_loader = f"model = SentenceTransformer({provider!r}{revision_arg})"
+        st_import = "from sentence_transformers import SentenceTransformer"
     item_prefix = metadata.get("item_prefix", "")
     text_prefix = metadata.get("text_prefix", "")
     text_column = metadata.get("text_column", "text")
@@ -78,7 +95,7 @@ import sys
 
 import numpy as np
 import pandas as pd
-from sentence_transformers import SentenceTransformer
+{st_import}
 
 TEXT_COLUMN = {text_column!r}
 ITEM_PREFIX = {item_prefix!r}   # model-required prefix (E5 family); empty = none
@@ -94,7 +111,7 @@ def main(csv_path: str) -> None:
     work = df.loc[mask].reset_index(drop=True)          # platform drops empty rows the same way
     texts = work[TEXT_COLUMN].astype(str).tolist()
 
-    model = SentenceTransformer({provider!r}{revision_arg})
+    {model_loader}
     item_texts = [ITEM_PREFIX + i["text"] for i in ITEMS]
     doc_texts = [TEXT_PREFIX + t for t in texts]
 

backend/tests/test_registry_and_prefixes.py

@@ -108,3 +108,43 @@ def test_model_language_unsupported_warning():
     assert w["code"] == "MODEL_LANGUAGE_UNSUPPORTED"
     assert model_language_warning("en", "m", frozenset({"en"}), None) is None
     assert model_language_warning("zz", "m", frozenset(), None) is None  # unknown coverage: no warning
+
+
+# ------------------------------------- psyembedding models (pooling fallback)
+def test_psyembedding_models_registered_with_pooling_fallback():
+    """The published PsyEmbedding repos lack the 1_Pooling config their
+    modules.json references, so plain SentenceTransformer(id) cannot load
+    them; the registry must flag them for explicit module assembly."""
+    from app import registry
+
+    psy = [m for m in registry.list_models() if m.id.startswith("psyembedding-")]
+    assert len(psy) == 4
+    for m in psy:
+        assert m.pooling_fallback == "mean", m.id
+        assert not m.requires_prefix, m.id
+        assert m.embedding_dimension == 1024 and m.max_seq_length == 512, m.id
+        assert m.lazy_load, m.id
+
+
+def test_repro_script_uses_explicit_modules_for_pooling_fallback():
+    from app.reproducibility import script_text
+
+    meta = {
+        "construct_snapshot": {"items": [{"text": "I am satisfied.", "reverse_scored": False}],
+                               "name": "X", "version": 1, "item_hash": "ab" * 16},
+        "model_registry_id": "psyembedding-bert-large",
+        "provider_model_id": "Culture-and-Morality-Lab/psyembedding-bert-large-uncased",
+        "model_revision": "PIN_ME",
+        "model_pooling_fallback": "mean",
+        "model_max_seq_length": 512,
+        "text_column": "text",
+    }
+    src = script_text(meta)
+    compile(src, "reproduce_analysis.py", "exec")
+    assert "st_models.Pooling" in src and "pooling_mode='mean'" in src
+    # plain models keep the one-line loader
+    plain = script_text({**meta, "model_pooling_fallback": None,
+                         "model_registry_id": "all-minilm-l6-v2",
+                         "provider_model_id": "sentence-transformers/all-MiniLM-L6-v2"})
+    compile(plain, "reproduce_analysis.py", "exec")
+    assert "st_models" not in plain

packages/model_registry/models.yaml

@@ -77,3 +77,127 @@ models:
     warnings:
       - "Quality varies by language, especially lower-resource languages."
       - "Scores are not comparable across model families."
+
+  # PsyEmbedding family (Culture & Morality Lab): sentence-transformers models
+  # fine-tuned for psychological text analysis on sentence-similarity pairs.
+  # Added on PI request 2026-07-18. All four: mean pooling, no prefix, English,
+  # 1024-dim, 512-token window, ~1.3 GB download on first use (lazy).
+
+  - id: psyembedding-bert-large
+    provider_model_id: Culture-and-Morality-Lab/psyembedding-bert-large-uncased
+    display_name: "PsyEmbedding BERT Large (psychology fine-tune)"
+    revision: PIN_ME
+    default: false
+    supported_languages: ["en"]
+    embedding_dimension: 1024
+    max_seq_length: 512
+    quality_tier: specialized
+    speed_tier: slow
+    usage_config:
+      requires_prefix: false
+      symmetric_similarity_prefix: null
+      pooling: mean                            # per model card; see pooling_fallback
+      # The published repos' modules.json references a 1_Pooling config the
+      # repos do not contain, so SentenceTransformer(id) cannot auto-load
+      # them; the backend builds Transformer+mean-Pooling explicitly.
+      pooling_fallback: mean
+      normalize_embeddings: true
+    operational_config:
+      expected_time_per_1k_texts_cpu: benchmark_required
+      worker_ram_min_gb: 4
+      lazy_load: true
+      max_resident_models: 1
+    warnings:
+      - "Fine-tuned for psychological text; general-domain performance untested."
+      - "Substantially slower and heavier than MiniLM."
+      - "Scores are not comparable across model families."
+
+  - id: psyembedding-roberta-large
+    provider_model_id: Culture-and-Morality-Lab/psyembedding-roberta-large
+    display_name: "PsyEmbedding RoBERTa Large (psychology fine-tune)"
+    revision: PIN_ME
+    default: false
+    supported_languages: ["en"]
+    embedding_dimension: 1024
+    max_seq_length: 512
+    quality_tier: specialized
+    speed_tier: slow
+    usage_config:
+      requires_prefix: false
+      symmetric_similarity_prefix: null
+      pooling: mean                            # per model card; see pooling_fallback
+      # The published repos' modules.json references a 1_Pooling config the
+      # repos do not contain, so SentenceTransformer(id) cannot auto-load
+      # them; the backend builds Transformer+mean-Pooling explicitly.
+      pooling_fallback: mean
+      normalize_embeddings: true
+    operational_config:
+      expected_time_per_1k_texts_cpu: benchmark_required
+      worker_ram_min_gb: 4
+      lazy_load: true
+      max_resident_models: 1
+    warnings:
+      - "Fine-tuned for psychological text; general-domain performance untested."
+      - "Substantially slower and heavier than MiniLM."
+      - "Scores are not comparable across model families."
+
+  - id: psyembedding-gte-large
+    provider_model_id: Culture-and-Morality-Lab/psyembedding-gte-large
+    display_name: "PsyEmbedding GTE Large (psychology fine-tune)"
+    revision: PIN_ME
+    default: false
+    supported_languages: ["en"]
+    embedding_dimension: 1024
+    max_seq_length: 512
+    quality_tier: specialized
+    speed_tier: slow
+    usage_config:
+      requires_prefix: false
+      symmetric_similarity_prefix: null
+      pooling: mean                            # per model card; see pooling_fallback
+      # The published repos' modules.json references a 1_Pooling config the
+      # repos do not contain, so SentenceTransformer(id) cannot auto-load
+      # them; the backend builds Transformer+mean-Pooling explicitly.
+      pooling_fallback: mean
+      normalize_embeddings: true
+    operational_config:
+      expected_time_per_1k_texts_cpu: benchmark_required
+      worker_ram_min_gb: 4
+      lazy_load: true
+      max_resident_models: 1
+    warnings:
+      - "Fine-tuned for psychological text; general-domain performance untested."
+      - "Substantially slower and heavier than MiniLM."
+      - "Scores are not comparable across model families."
+
+  - id: psyembedding-e5-large-v2
+    provider_model_id: Culture-and-Morality-Lab/psyembedding-e5-large-v2
+    display_name: "PsyEmbedding E5 Large v2 (psychology fine-tune)"
+    revision: PIN_ME
+    default: false
+    supported_languages: ["en"]
+    embedding_dimension: 1024
+    max_seq_length: 512
+    quality_tier: specialized
+    speed_tier: slow
+    usage_config:
+      # Unlike base intfloat/e5-large-v2, the fine-tune's card specifies direct
+      # encoding with NO "query: " prefix; the ST package carries its own
+      # pooling+normalize config, so we match the card exactly.
+      requires_prefix: false
+      symmetric_similarity_prefix: null
+      pooling: mean                            # per model card; see pooling_fallback
+      # The published repos' modules.json references a 1_Pooling config the
+      # repos do not contain, so SentenceTransformer(id) cannot auto-load
+      # them; the backend builds Transformer+mean-Pooling explicitly.
+      pooling_fallback: mean
+      normalize_embeddings: true
+    operational_config:
+      expected_time_per_1k_texts_cpu: benchmark_required
+      worker_ram_min_gb: 4
+      lazy_load: true
+      max_resident_models: 1
+    warnings:
+      - "Fine-tuned for psychological text; general-domain performance untested."
+      - "Substantially slower and heavier than MiniLM."
+      - "Scores are not comparable across model families."