Spaces:
Sleeping
Sleeping
File size: 6,002 Bytes
57a9a08 285b041 57a9a08 9178ee1 57a9a08 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | """Concept-vector search: nearest papers to added concepts minus removed ones.
The document-level analogue of ``king − man + woman``. Each concept is embedded
once, the vectors are combined into one direction, and the index is searched by
nearest neighbor to it. Every match carries its cosine to each supplied concept
so the answer stays legible: what pulled a paper in, and what should have pushed
it away. Nothing here scores novelty or likelihood — only cosines and counts.
"""
from __future__ import annotations
import asyncio
import numpy as np
from app.config import Settings, settings
from app.embeddings import QueryEmbedder
from app.gapmap import _bucket, combine
from app.models import ConceptSearchRequest
VERSION = "concepts-v1"
def aim(
query: list[float], positive: list[list[float]], negative: list[list[float]]
) -> list[float] | None:
"""Point a question's own vector at the concepts kept and away from those pushed off.
The question is simply one more positive term, so a tagged search is the
same search: retrieval still matches the question's keywords, and the tags
only move the vector half of the ranking. ``None`` when the terms cancel the
question out, which has no direction to search in.
"""
combined = combine(
[np.asarray(query, dtype=np.float32), *(np.asarray(v, dtype=np.float32) for v in positive)],
[np.asarray(v, dtype=np.float32) for v in negative],
)
return None if combined is None else [float(value) for value in combined]
class ConceptError(Exception):
"""A concept search that cannot honestly return matches."""
status = 503
class EmbeddingsUnavailable(ConceptError):
status = 503
class ConceptsCancelOut(ConceptError):
status = 422
def _unit(values) -> np.ndarray | None:
vector = np.asarray(values, dtype=np.float32)
norm = float(np.linalg.norm(vector))
return None if norm < 1e-6 else vector / norm
def _source(document: dict) -> str:
source = document.get("source") or "openalex"
return "clinicaltrials" if source == "ctgov" else str(source)
class ConceptVectorService:
def __init__(self, repository, config: Settings = settings):
self.repo = repository
self.config = config
self.embedder = QueryEmbedder(config)
async def embed(self, text: str) -> list[float] | None:
return await self.embedder.embed(text)
async def resolve(
self, positive: list[str], negative: list[str]
) -> tuple[np.ndarray, dict[str, np.ndarray]]:
"""Embed each distinct concept once and combine them into one direction.
The per-term vectors come back with the combination so a caller can
explain a result against each concept, or reuse the direction for a
query of its own without paying for the embeddings twice.
"""
terms = list(dict.fromkeys([*positive, *negative]))
try:
vectors = await asyncio.gather(*(self.embed(term) for term in terms))
except Exception as exc:
raise EmbeddingsUnavailable(
f"The embedding model could not be loaded ({type(exc).__name__}), "
"so the concepts could not be turned into vectors."
) from exc
if any(vector is None for vector in vectors):
raise EmbeddingsUnavailable(
"Embeddings are disabled, so the concepts could not be turned into vectors."
)
by_term = {
term: np.asarray(vector, dtype=np.float32) for term, vector in zip(terms, vectors)
}
combined = combine(
[by_term[term] for term in positive], [by_term[term] for term in negative]
)
if combined is None:
raise ConceptsCancelOut(
"The concepts cancel each other out, so the search has no direction."
)
return combined, by_term
async def search(self, request: ConceptSearchRequest) -> dict:
combined, by_term = await self.resolve(request.positive, request.negative)
filters = (
request.filters.model_dump()
if request.filters is not None and request.filters.active
else None
)
hits = await self.repo.knn_studies(
[float(value) for value in combined], limit=request.limit, filters=filters
)
signed = [(term, "positive") for term in request.positive]
signed += [(term, "negative") for term in request.negative]
warnings: list[str] = []
matches = []
for hit in hits:
vector = _unit(hit.get("embedding") or [])
if vector is None and not warnings:
warnings.append(
"Some matches were stored without a vector, so their per-concept "
"cosines are missing."
)
matches.append(
{
"id": str(hit.get("id", "")),
"title": hit.get("title", ""),
"year": hit.get("year") if isinstance(hit.get("year"), int) else None,
"url": hit.get("url") or "",
"source": _source(hit),
"verdict": _bucket(hit),
"citations": hit.get("cited_by_count") or 0,
"cosine": round(float(hit.get("cosine", 0.0)), 4),
"concepts": [
{
"text": term,
"sign": sign,
"cosine": round(float(vector @ by_term[term]), 4),
}
for term, sign in signed
]
if vector is not None
else [],
}
)
return {
"version": VERSION,
"concepts": [{"text": term, "sign": sign} for term, sign in signed],
"matches": matches,
"warnings": warnings,
}
|