Spaces:
Sleeping
Sleeping
| """Concept-vector search: nearest papers to added concepts minus removed ones. | |
| The document-level analogue of ``king − man + woman``. Each concept is embedded | |
| once, the vectors are combined into one direction, and the index is searched by | |
| nearest neighbor to it. Every match carries its cosine to each supplied concept | |
| so the answer stays legible: what pulled a paper in, and what should have pushed | |
| it away. Nothing here scores novelty or likelihood — only cosines and counts. | |
| """ | |
| from __future__ import annotations | |
| import asyncio | |
| import numpy as np | |
| from app.config import Settings, settings | |
| from app.embeddings import QueryEmbedder | |
| from app.gapmap import _bucket, combine | |
| from app.models import ConceptSearchRequest | |
| VERSION = "concepts-v1" | |
| def aim( | |
| query: list[float], positive: list[list[float]], negative: list[list[float]] | |
| ) -> list[float] | None: | |
| """Point a question's own vector at the concepts kept and away from those pushed off. | |
| The question is simply one more positive term, so a tagged search is the | |
| same search: retrieval still matches the question's keywords, and the tags | |
| only move the vector half of the ranking. ``None`` when the terms cancel the | |
| question out, which has no direction to search in. | |
| """ | |
| combined = combine( | |
| [np.asarray(query, dtype=np.float32), *(np.asarray(v, dtype=np.float32) for v in positive)], | |
| [np.asarray(v, dtype=np.float32) for v in negative], | |
| ) | |
| return None if combined is None else [float(value) for value in combined] | |
| class ConceptError(Exception): | |
| """A concept search that cannot honestly return matches.""" | |
| status = 503 | |
| class EmbeddingsUnavailable(ConceptError): | |
| status = 503 | |
| class ConceptsCancelOut(ConceptError): | |
| status = 422 | |
| def _unit(values) -> np.ndarray | None: | |
| vector = np.asarray(values, dtype=np.float32) | |
| norm = float(np.linalg.norm(vector)) | |
| return None if norm < 1e-6 else vector / norm | |
| def _source(document: dict) -> str: | |
| source = document.get("source") or "openalex" | |
| return "clinicaltrials" if source == "ctgov" else str(source) | |
| class ConceptVectorService: | |
| def __init__(self, repository, config: Settings = settings): | |
| self.repo = repository | |
| self.config = config | |
| self.embedder = QueryEmbedder(config) | |
| async def embed(self, text: str) -> list[float] | None: | |
| return await self.embedder.embed(text) | |
| async def resolve( | |
| self, positive: list[str], negative: list[str] | |
| ) -> tuple[np.ndarray, dict[str, np.ndarray]]: | |
| """Embed each distinct concept once and combine them into one direction. | |
| The per-term vectors come back with the combination so a caller can | |
| explain a result against each concept, or reuse the direction for a | |
| query of its own without paying for the embeddings twice. | |
| """ | |
| terms = list(dict.fromkeys([*positive, *negative])) | |
| try: | |
| vectors = await asyncio.gather(*(self.embed(term) for term in terms)) | |
| except Exception as exc: | |
| raise EmbeddingsUnavailable( | |
| f"The embedding model could not be loaded ({type(exc).__name__}), " | |
| "so the concepts could not be turned into vectors." | |
| ) from exc | |
| if any(vector is None for vector in vectors): | |
| raise EmbeddingsUnavailable( | |
| "Embeddings are disabled, so the concepts could not be turned into vectors." | |
| ) | |
| by_term = { | |
| term: np.asarray(vector, dtype=np.float32) for term, vector in zip(terms, vectors) | |
| } | |
| combined = combine( | |
| [by_term[term] for term in positive], [by_term[term] for term in negative] | |
| ) | |
| if combined is None: | |
| raise ConceptsCancelOut( | |
| "The concepts cancel each other out, so the search has no direction." | |
| ) | |
| return combined, by_term | |
| async def search(self, request: ConceptSearchRequest) -> dict: | |
| combined, by_term = await self.resolve(request.positive, request.negative) | |
| filters = ( | |
| request.filters.model_dump() | |
| if request.filters is not None and request.filters.active | |
| else None | |
| ) | |
| hits = await self.repo.knn_studies( | |
| [float(value) for value in combined], limit=request.limit, filters=filters | |
| ) | |
| signed = [(term, "positive") for term in request.positive] | |
| signed += [(term, "negative") for term in request.negative] | |
| warnings: list[str] = [] | |
| matches = [] | |
| for hit in hits: | |
| vector = _unit(hit.get("embedding") or []) | |
| if vector is None and not warnings: | |
| warnings.append( | |
| "Some matches were stored without a vector, so their per-concept " | |
| "cosines are missing." | |
| ) | |
| matches.append( | |
| { | |
| "id": str(hit.get("id", "")), | |
| "title": hit.get("title", ""), | |
| "year": hit.get("year") if isinstance(hit.get("year"), int) else None, | |
| "url": hit.get("url") or "", | |
| "source": _source(hit), | |
| "verdict": _bucket(hit), | |
| "citations": hit.get("cited_by_count") or 0, | |
| "cosine": round(float(hit.get("cosine", 0.0)), 4), | |
| "concepts": [ | |
| { | |
| "text": term, | |
| "sign": sign, | |
| "cosine": round(float(vector @ by_term[term]), 4), | |
| } | |
| for term, sign in signed | |
| ] | |
| if vector is not None | |
| else [], | |
| } | |
| ) | |
| return { | |
| "version": VERSION, | |
| "concepts": [{"text": term, "sign": sign} for term, sign in signed], | |
| "matches": matches, | |
| "warnings": warnings, | |
| } | |