Upload folder using huggingface_hub

app/concepts.py

@@ -21,6 +21,23 @@ from app.models import ConceptSearchRequest
 VERSION = "concepts-v1"
 
 
+def aim(
+    query: list[float], positive: list[list[float]], negative: list[list[float]]
+) -> list[float] | None:
+    """Point a question's own vector at the concepts kept and away from those pushed off.
+
+    The question is simply one more positive term, so a tagged search is the
+    same search: retrieval still matches the question's keywords, and the tags
+    only move the vector half of the ranking. ``None`` when the terms cancel the
+    question out, which has no direction to search in.
+    """
+    combined = combine(
+        [np.asarray(query, dtype=np.float32), *(np.asarray(v, dtype=np.float32) for v in positive)],
+        [np.asarray(v, dtype=np.float32) for v in negative],
+    )
+    return None if combined is None else [float(value) for value in combined]
+
+
 class ConceptError(Exception):
     """A concept search that cannot honestly return matches."""
 

app/models.py

@@ -4,6 +4,8 @@ from typing import Annotated, Literal
 
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
+ConceptTerm = Annotated[str, Field(min_length=2, max_length=200)]
+
 Bucket = Literal[
     "effect", "credible_null", "reported_null", "inconclusive", "failed", "unreported"
 ]
@@ -50,6 +52,24 @@ class SearchFilters(BaseModel):
         return self.yearFrom is not None or self.yearTo is not None
 
 
+class ConceptSteer(BaseModel):
+    """Concepts that aim the same search, rather than a search of their own.
+
+    Each term is one more direction added to (or subtracted from) the question's
+    own vector before retrieval, so the tags change what ranks highest among the
+    question's matches. Subtracting is a direction, not a filter: a paper is
+    pushed down for being about the term, never removed for mentioning it.
+    """
+
+    model_config = ConfigDict(extra="forbid", str_strip_whitespace=True)
+    positive: list[ConceptTerm] = Field(default_factory=list, max_length=8)
+    negative: list[ConceptTerm] = Field(default_factory=list, max_length=8)
+
+    @property
+    def active(self) -> bool:
+        return bool(self.positive or self.negative)
+
+
 class SearchRequest(BaseModel):
     model_config = ConfigDict(extra="forbid", str_strip_whitespace=True, allow_inf_nan=False)
     idea: str = Field(min_length=8, max_length=12000)
@@ -64,6 +84,7 @@ class SearchRequest(BaseModel):
     outcomeSd: float | None = Field(default=None, gt=0, le=1e6)
     baselineRisk: float | None = Field(default=None, gt=0, lt=1)
     filters: SearchFilters | None = None
+    concepts: ConceptSteer | None = None
 
 
 class Pico(BaseModel):
@@ -240,9 +261,6 @@ class MapRequest(BaseModel):
     refresh: bool = False
 
 
-ConceptTerm = Annotated[str, Field(min_length=2, max_length=200)]
-
-
 class ConceptSearchRequest(BaseModel):
     """Concepts to add and to subtract before the index is searched by nearest neighbour.
 

app/pipeline.py

@@ -8,10 +8,11 @@ from datetime import UTC, datetime
 from time import perf_counter
 from uuid import uuid4
 
+from app.concepts import aim
 from app.config import Settings, settings
 from app.fulltext import FullTextClient
 from app.llm import LLMService, Usage
-from app.models import Pico, SearchRequest
+from app.models import ConceptSteer, Pico, SearchRequest
 from app.repository import (
     BUCKETS,
     EFFECT_DIRECTIONS,
@@ -221,6 +222,45 @@ class SearchPipeline:
             self.embedder = get_embedder(self.config.embedding_model, self.config.embedding_device)
         return await asyncio.to_thread(self.embedder.embed_query, text)
 
+    async def aim(
+        self, vector: list[float] | None, steer: ConceptSteer, warnings: list[str]
+    ) -> list[float] | None:
+        """Move the question's vector towards the kept concepts and away from the pushed ones.
+
+        The tags change ranking, never the match set: the lexical query is
+        untouched, so a search with tags returns the question's studies in a
+        different order rather than a different search's studies.
+        """
+        if vector is None:
+            warnings.append(
+                "Concept tags need embeddings to steer ranking, and embeddings are "
+                "unavailable here, so the tags were ignored."
+            )
+            return None
+        terms = list(dict.fromkeys([*steer.positive, *steer.negative]))
+        try:
+            embedded = await asyncio.gather(*(self.embed(term) for term in terms))
+        except Exception as exc:
+            logger.warning("Concept embedding unavailable: %s", type(exc).__name__)
+            warnings.append("The concept tags could not be embedded, so ranking was not steered.")
+            return vector
+        by_term = dict(zip(terms, embedded))
+        if any(by_term[term] is None for term in terms):
+            warnings.append("The concept tags could not be embedded, so ranking was not steered.")
+            return vector
+        steered = aim(
+            vector,
+            [by_term[term] for term in steer.positive],
+            [by_term[term] for term in steer.negative],
+        )
+        if steered is None:
+            warnings.append(
+                "The concept tags cancelled the question out, leaving no direction to "
+                "search in, so ranking was not steered."
+            )
+            return vector
+        return steered
+
     async def expand(
         self,
         hits: list[dict],
@@ -767,6 +807,9 @@ class SearchPipeline:
             vector = None
             logger.warning("Local embeddings unavailable: %s", type(exc).__name__)
             warnings.append("Local embeddings are unavailable; retrieval used BM25 keyword search.")
+        steer = request.concepts if request.concepts and request.concepts.active else None
+        if steer:
+            vector = await self.aim(vector, steer, warnings)
         hits, mode = await self.repo.retrieve(query, vector)
         expanded = await self.expand(hits, vector, warnings, pico.model_dump(), filters)
         expansion_ids = [d["id"] for d in expanded]
@@ -993,6 +1036,8 @@ class SearchPipeline:
             "retrieval": {
                 "mode": mode,
                 "expanded": len(expanded),
+                # Echoed so a reordered page can say what reordered it.
+                "concepts": steer.model_dump() if steer else None,
                 "durationMs": round((perf_counter() - started) * 1000),
             },
             "spin": aggregation["spin"],