Update src/api/search.py
Browse files- src/api/search.py +219 -336
src/api/search.py
CHANGED
|
@@ -1,25 +1,22 @@
|
|
| 1 |
"""
|
| 2 |
-
search.py β
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
|
|
|
| 9 |
"""
|
| 10 |
-
import
|
| 11 |
-
from
|
|
|
|
| 12 |
from urllib.parse import quote
|
| 13 |
|
| 14 |
-
import
|
| 15 |
-
from fastapi import APIRouter
|
| 16 |
-
from pydantic import BaseModel, field_validator
|
| 17 |
|
| 18 |
-
from src.config import get_settings
|
| 19 |
-
|
| 20 |
-
logger = logging.getLogger(__name__)
|
| 21 |
-
router = APIRouter()
|
| 22 |
|
|
|
|
| 23 |
|
| 24 |
class SearchRequest(BaseModel):
|
| 25 |
query: str
|
|
@@ -29,308 +26,237 @@ class SearchRequest(BaseModel):
|
|
| 29 |
open_access: bool = False
|
| 30 |
year_from: Optional[str] = None
|
| 31 |
year_to: Optional[str] = None
|
| 32 |
-
|
| 33 |
-
resource_type: str = "articles" # "articles" | "books" | "both"
|
| 34 |
-
|
| 35 |
-
@field_validator("query")
|
| 36 |
-
@classmethod
|
| 37 |
-
def query_not_empty(cls, v: str) -> str:
|
| 38 |
-
v = v.strip()
|
| 39 |
-
if not v:
|
| 40 |
-
raise ValueError("query must not be empty")
|
| 41 |
-
if len(v) > 500:
|
| 42 |
-
raise ValueError("query must be under 500 characters")
|
| 43 |
-
return v
|
| 44 |
-
|
| 45 |
-
@field_validator("limit")
|
| 46 |
-
@classmethod
|
| 47 |
-
def limit_in_range(cls, v: int) -> int:
|
| 48 |
-
if not 1 <= v <= 20:
|
| 49 |
-
raise ValueError("limit must be between 1 and 20")
|
| 50 |
-
return v
|
| 51 |
-
|
| 52 |
-
@field_validator("source")
|
| 53 |
-
@classmethod
|
| 54 |
-
def valid_source(cls, v: str) -> str:
|
| 55 |
-
allowed = {"primo", "pubmed", "scholar"}
|
| 56 |
-
if v not in allowed:
|
| 57 |
-
raise ValueError(f"source must be one of: {allowed}")
|
| 58 |
-
return v
|
| 59 |
-
|
| 60 |
-
@field_validator("resource_type")
|
| 61 |
-
@classmethod
|
| 62 |
-
def valid_resource_type(cls, v: str) -> str:
|
| 63 |
-
allowed = {"articles", "books", "both"}
|
| 64 |
-
if v not in allowed:
|
| 65 |
-
return "articles"
|
| 66 |
-
return v
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
def _safe_metrics_increment(key: str) -> None:
|
| 70 |
-
try:
|
| 71 |
-
from app import metrics_service
|
| 72 |
-
metrics_service.incr(key)
|
| 73 |
-
except Exception:
|
| 74 |
-
return
|
| 75 |
|
| 76 |
|
| 77 |
-
|
| 78 |
-
try:
|
| 79 |
-
from app import metrics_service
|
| 80 |
-
metrics_service.incr_bucket(bucket, key)
|
| 81 |
-
except Exception:
|
| 82 |
-
return
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
@router.post("")
|
| 86 |
-
async def search_endpoint(req: SearchRequest):
|
| 87 |
-
_safe_metrics_increment("search_requests")
|
| 88 |
-
_safe_metrics_bucket("search_sources", req.source)
|
| 89 |
-
settings = get_settings()
|
| 90 |
-
if req.source == "primo":
|
| 91 |
-
return await search_primo(req, settings.primo_api_key)
|
| 92 |
-
if req.source == "pubmed":
|
| 93 |
-
return await search_pubmed(req)
|
| 94 |
-
return await search_scholar(req)
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
# ββ PRIMO ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 98 |
-
|
| 99 |
-
# ββ PRIMO helpers βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 100 |
-
|
| 101 |
-
def _first_str(value):
|
| 102 |
-
if isinstance(value, list) and value:
|
| 103 |
-
return value[0]
|
| 104 |
-
if isinstance(value, str):
|
| 105 |
-
return value
|
| 106 |
-
return ""
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
def _extract_primo_docid(doc: dict) -> str:
|
| 110 |
-
pnx = doc.get("pnx", {}) or {}
|
| 111 |
-
control = pnx.get("control", {}) or {}
|
| 112 |
-
search_sec = pnx.get("search", {}) or {}
|
| 113 |
-
candidates: List[str] = [
|
| 114 |
-
_first_str(control.get("recordid")),
|
| 115 |
-
_first_str(control.get("sourcerecordid")),
|
| 116 |
-
_first_str(search_sec.get("recordid")),
|
| 117 |
-
_first_str(search_sec.get("addsrcrecordid")),
|
| 118 |
-
]
|
| 119 |
-
for key in ("recordid", "pnxId", "id"):
|
| 120 |
-
val = doc.get(key)
|
| 121 |
-
if isinstance(val, str):
|
| 122 |
-
candidates.append(val)
|
| 123 |
-
elif isinstance(val, list) and val:
|
| 124 |
-
candidates.append(val[0])
|
| 125 |
-
for cand in candidates:
|
| 126 |
-
cand = (cand or "").strip()
|
| 127 |
-
if cand:
|
| 128 |
-
return cand
|
| 129 |
-
return ""
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
def _primo_record_url(doc: dict, query_encoded: str) -> str:
|
| 133 |
-
delivery = doc.get("delivery", {}) or {}
|
| 134 |
-
bestlocation = delivery.get("bestlocation", {}) or {}
|
| 135 |
-
for key in ("link", "url"):
|
| 136 |
-
val = bestlocation.get(key)
|
| 137 |
-
if isinstance(val, str) and "fulldisplay" in val:
|
| 138 |
-
return val
|
| 139 |
-
links = delivery.get("links", []) or []
|
| 140 |
-
for link_obj in links:
|
| 141 |
-
if isinstance(link_obj, dict):
|
| 142 |
-
val = link_obj.get("link") or link_obj.get("url") or ""
|
| 143 |
-
if isinstance(val, str) and "fulldisplay" in val:
|
| 144 |
-
return val
|
| 145 |
-
docid = _extract_primo_docid(doc)
|
| 146 |
-
if docid:
|
| 147 |
-
pnx = doc.get("pnx", {}) or {}
|
| 148 |
-
control = pnx.get("control", {}) or {}
|
| 149 |
-
sourceid = _first_str(control.get("sourceid")).lower()
|
| 150 |
-
context = "PC" if docid.lower().startswith("cdi_") or "central" in sourceid else "L"
|
| 151 |
-
return (
|
| 152 |
-
"https://khalifa.primo.exlibrisgroup.com/discovery/fulldisplay"
|
| 153 |
-
f"?docid={quote(docid)}&context={context}&vid=971KUOSTAR_INST:KU&lang=en&search_scope=MyInst_and_CI&tab=Everything"
|
| 154 |
-
)
|
| 155 |
-
return (
|
| 156 |
-
"https://khalifa.primo.exlibrisgroup.com/discovery/search"
|
| 157 |
-
f"?query=any,contains,{query_encoded}&vid=971KUOSTAR_INST:KU&lang=en&search_scope=MyInst_and_CI&tab=Everything"
|
| 158 |
-
)
|
| 159 |
|
| 160 |
-
|
| 161 |
-
"""
|
| 162 |
-
Send clean topic keywords to q=any,contains.
|
| 163 |
-
Resource type, peer review, open access, and year are all FACETS β never keywords.
|
| 164 |
-
|
| 165 |
-
PRIMO facet reference:
|
| 166 |
-
facet_rtype,include,articles β journal articles
|
| 167 |
-
facet_rtype,include,books β books / ebooks
|
| 168 |
-
facet_tlevel,include,peer_reviewed
|
| 169 |
-
facet_tlevel,include,online_resources (open access proxy)
|
| 170 |
-
facet_searchcreationdate,include,YYYY|,|YYYY
|
| 171 |
"""
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
_safe_metrics_bucket("errors", "primo_missing_api_key")
|
| 175 |
-
return {"error": "PRIMO_API_KEY not configured", "results": [], "source": "PRIMO"}
|
| 176 |
-
|
| 177 |
-
# ββ The query sent to PRIMO must be ONLY topic keywords ββββββββββββββββββ
|
| 178 |
-
# Strip any residual resource-type words that may have slipped through
|
| 179 |
-
# topic extraction (belt-and-braces defence).
|
| 180 |
-
clean_query = _strip_resource_noise(req.query)
|
| 181 |
-
if not clean_query:
|
| 182 |
-
clean_query = req.query # fallback if stripping removes everything
|
| 183 |
-
|
| 184 |
-
query_encoded = quote(clean_query)
|
| 185 |
-
|
| 186 |
-
url = (
|
| 187 |
-
f"https://api-eu.hosted.exlibrisgroup.com/primo/v1/search"
|
| 188 |
-
f"?vid=971KUOSTAR_INST:KU&tab=Everything&scope=MyInst_and_CI"
|
| 189 |
-
f"&q=any,contains,{query_encoded}&lang=en&sort=rank&limit={req.limit}"
|
| 190 |
-
f"&offset=0&mode=advanced&apikey={api_key}"
|
| 191 |
-
)
|
| 192 |
-
|
| 193 |
-
# ββ Build facets β resource type, quality, year βββββββββββββββββββββββββββ
|
| 194 |
-
facets = []
|
| 195 |
-
|
| 196 |
-
# Resource type β rtype facet (NOT a keyword)
|
| 197 |
-
if req.resource_type == "articles":
|
| 198 |
-
facets.append("facet_rtype,include,articles")
|
| 199 |
-
elif req.resource_type == "books":
|
| 200 |
-
facets.append("facet_rtype,include,books")
|
| 201 |
-
# "both" β no rtype facet (search everything)
|
| 202 |
|
| 203 |
-
|
| 204 |
-
|
|
|
|
| 205 |
|
| 206 |
-
|
| 207 |
-
|
| 208 |
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
facets.append(f"facet_searchcreationdate,include,{yf}|,|{yt}")
|
| 213 |
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
)
|
| 221 |
|
| 222 |
-
try:
|
| 223 |
-
async with httpx.AsyncClient(timeout=20) as client:
|
| 224 |
-
r = await client.get(url, headers={"Accept": "application/json"})
|
| 225 |
-
logger.info("PRIMO status=%s", r.status_code)
|
| 226 |
-
if r.status_code != 200:
|
| 227 |
-
_safe_metrics_bucket("errors", f"primo_{r.status_code}")
|
| 228 |
-
return {"error": f"PRIMO API error: {r.status_code}", "results": [], "source": "PRIMO"}
|
| 229 |
|
| 230 |
-
|
| 231 |
-
docs = data.get("docs", [])
|
| 232 |
-
results = []
|
| 233 |
-
for doc in docs[: req.limit]:
|
| 234 |
-
pnx = doc.get("pnx", {})
|
| 235 |
-
d = pnx.get("display", {})
|
| 236 |
-
results.append({
|
| 237 |
-
"title": (d.get("title") or ["Untitled"])[0],
|
| 238 |
-
"creator": "; ".join(d.get("creator") or d.get("contributor") or []) or "Unknown",
|
| 239 |
-
"date": (d.get("date") or [""])[0],
|
| 240 |
-
"type": (d.get("type") or [""])[0],
|
| 241 |
-
"source": (d.get("source") or [""])[0],
|
| 242 |
-
"primo_url": _primo_record_url(doc, query_encoded),
|
| 243 |
-
"_source": "PRIMO",
|
| 244 |
-
})
|
| 245 |
-
return {
|
| 246 |
-
"total": data.get("info", {}).get("total", 0),
|
| 247 |
-
"results": results,
|
| 248 |
-
"source": "PRIMO",
|
| 249 |
-
}
|
| 250 |
-
|
| 251 |
-
except httpx.TimeoutException:
|
| 252 |
-
_safe_metrics_bucket("errors", "primo_timeout")
|
| 253 |
-
return {"error": "PRIMO API timed out β please try again", "results": [], "source": "PRIMO"}
|
| 254 |
-
except Exception as exc:
|
| 255 |
-
logger.error("PRIMO search error: %s", exc, exc_info=True)
|
| 256 |
-
_safe_metrics_bucket("errors", "primo_exception")
|
| 257 |
-
return {"error": str(exc), "results": [], "source": "PRIMO"}
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
def _strip_resource_noise(query: str) -> str:
|
| 261 |
"""
|
| 262 |
-
|
| 263 |
-
|
| 264 |
"""
|
| 265 |
-
|
| 266 |
-
r"\b(articles?|papers?|books?|ebooks?|journals?|publications?|monographs?"
|
| 267 |
-
r"|peer[- ]reviewed|open[- ]access|studies|study|research)\b",
|
| 268 |
-
re.IGNORECASE,
|
| 269 |
-
)
|
| 270 |
-
cleaned = noise.sub(" ", query)
|
| 271 |
-
# Collapse leftover AND/OR operators with no operands
|
| 272 |
-
cleaned = re.sub(r"\s+(AND|OR)\s+(AND|OR)\s+", " AND ", cleaned, flags=re.IGNORECASE)
|
| 273 |
-
cleaned = re.sub(r"^\s*(AND|OR)\s+", "", cleaned, flags=re.IGNORECASE)
|
| 274 |
-
cleaned = re.sub(r"\s+(AND|OR)\s*$", "", cleaned, flags=re.IGNORECASE)
|
| 275 |
-
return re.sub(r"\s+", " ", cleaned).strip()
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
import re # noqa: E402 (needed for _strip_resource_noise above)
|
| 279 |
|
|
|
|
|
|
|
| 280 |
|
| 281 |
-
#
|
|
|
|
|
|
|
| 282 |
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
clean_query = _strip_resource_noise(req.query)
|
| 290 |
-
if not clean_query:
|
| 291 |
-
clean_query = req.query
|
| 292 |
|
| 293 |
-
#
|
| 294 |
-
|
| 295 |
if req.peer_reviewed:
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
# Date params (separate from search term β correct PubMed approach)
|
| 301 |
-
date_params = ""
|
| 302 |
if req.year_from or req.year_to:
|
| 303 |
yf = req.year_from or "1900"
|
| 304 |
-
yt = req.year_to or
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
)
|
| 311 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 312 |
try:
|
| 313 |
async with httpx.AsyncClient(timeout=15) as client:
|
| 314 |
-
|
| 315 |
-
f"
|
| 316 |
-
|
|
|
|
|
|
|
|
|
|
| 317 |
)
|
| 318 |
-
r = await client.get(search_url)
|
| 319 |
if r.status_code != 200:
|
| 320 |
-
_safe_metrics_bucket("errors", "pubmed_search_failed")
|
| 321 |
return {"error": "PubMed search failed", "results": [], "source": "PubMed"}
|
| 322 |
|
| 323 |
ids = r.json().get("esearchresult", {}).get("idlist", [])
|
|
|
|
| 324 |
if not ids:
|
| 325 |
return {"total": 0, "results": [], "source": "PubMed"}
|
| 326 |
|
| 327 |
-
|
| 328 |
-
f"
|
| 329 |
-
|
| 330 |
)
|
| 331 |
-
r2 = await client.get(fetch_url)
|
| 332 |
if r2.status_code != 200:
|
| 333 |
-
_safe_metrics_bucket("errors", "pubmed_fetch_failed")
|
| 334 |
return {"error": "PubMed fetch failed", "results": [], "source": "PubMed"}
|
| 335 |
|
| 336 |
data = r2.json().get("result", {})
|
|
@@ -339,67 +265,24 @@ async def search_pubmed(req: SearchRequest):
|
|
| 339 |
rec = data.get(pid, {})
|
| 340 |
if not isinstance(rec, dict):
|
| 341 |
continue
|
| 342 |
-
|
| 343 |
-
authors = ", ".join(a.get("name", "") for a in
|
| 344 |
-
if len(
|
| 345 |
authors += " et al."
|
|
|
|
|
|
|
| 346 |
results.append({
|
| 347 |
"title": rec.get("title", ""),
|
| 348 |
"creator": authors,
|
| 349 |
"date": rec.get("pubdate", ""),
|
| 350 |
"source": rec.get("fulljournalname", rec.get("source", "")),
|
|
|
|
| 351 |
"pmid": pid,
|
| 352 |
"link": f"https://pubmed.ncbi.nlm.nih.gov/{pid}/",
|
| 353 |
"type": "Journal Article",
|
| 354 |
"_source": "PubMed",
|
| 355 |
})
|
| 356 |
-
return {"total":
|
| 357 |
-
|
| 358 |
-
except Exception as exc:
|
| 359 |
-
logger.error("PubMed search error: %s", exc)
|
| 360 |
-
_safe_metrics_bucket("errors", "pubmed_exception")
|
| 361 |
-
return {"error": str(exc), "results": [], "source": "PubMed"}
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
# ββ Semantic Scholar βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 365 |
-
|
| 366 |
-
async def search_scholar(req: SearchRequest):
|
| 367 |
-
try:
|
| 368 |
-
params = {
|
| 369 |
-
"query": req.query,
|
| 370 |
-
"limit": req.limit,
|
| 371 |
-
"fields": "title,authors,year,venue,externalIds,citationCount",
|
| 372 |
-
}
|
| 373 |
-
async with httpx.AsyncClient(timeout=15) as client:
|
| 374 |
-
r = await client.get(
|
| 375 |
-
"https://api.semanticscholar.org/graph/v1/paper/search",
|
| 376 |
-
params=params,
|
| 377 |
-
)
|
| 378 |
-
if r.status_code != 200:
|
| 379 |
-
_safe_metrics_bucket("errors", f"semantic_{r.status_code}")
|
| 380 |
-
return {"error": f"Semantic Scholar API: {r.status_code}", "results": [], "source": "Semantic Scholar"}
|
| 381 |
-
|
| 382 |
-
data = r.json()
|
| 383 |
-
results = []
|
| 384 |
-
for paper in data.get("data", []):
|
| 385 |
-
author_list = paper.get("authors") or []
|
| 386 |
-
authors = ", ".join(a.get("name", "") for a in author_list[:3])
|
| 387 |
-
if len(author_list) > 3:
|
| 388 |
-
authors += " et al."
|
| 389 |
-
paper_id = paper.get("paperId")
|
| 390 |
-
results.append({
|
| 391 |
-
"title": paper.get("title", ""),
|
| 392 |
-
"creator": authors,
|
| 393 |
-
"date": str(paper.get("year", "")),
|
| 394 |
-
"source": paper.get("venue", ""),
|
| 395 |
-
"citations": paper.get("citationCount", 0),
|
| 396 |
-
"link": f"https://www.semanticscholar.org/paper/{paper_id}" if paper_id else "",
|
| 397 |
-
"type": "Article",
|
| 398 |
-
"_source": "Semantic Scholar",
|
| 399 |
-
})
|
| 400 |
-
return {"total": data.get("total", 0), "results": results, "source": "Semantic Scholar"}
|
| 401 |
|
| 402 |
-
except Exception as
|
| 403 |
-
|
| 404 |
-
_safe_metrics_bucket("errors", "semantic_exception")
|
| 405 |
-
return {"error": str(exc), "results": [], "source": "Semantic Scholar"}
|
|
|
|
| 1 |
"""
|
| 2 |
+
src/api/search.py β PRIMO + PubMed search helpers for LibBee.
|
| 3 |
+
|
| 4 |
+
Key fix: _boolean_to_primo_params() converts an LLM-generated Boolean query like
|
| 5 |
+
("smart city" OR "urban mobility") AND ("technocentricity" OR "citizen-centric")
|
| 6 |
+
into PRIMO's multi-q-param format:
|
| 7 |
+
q=any,contains,smart city OR urban mobility,AND&q=any,contains,technocentricity OR citizen-centric
|
| 8 |
+
This is the only format PRIMO accepts for structured boolean searching.
|
| 9 |
+
Sending the raw boolean string as a single q= param causes irrelevant results.
|
| 10 |
"""
|
| 11 |
+
import re
|
| 12 |
+
from datetime import datetime
|
| 13 |
+
from typing import Optional
|
| 14 |
from urllib.parse import quote
|
| 15 |
|
| 16 |
+
from pydantic import BaseModel
|
|
|
|
|
|
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
+
# ββ Models βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 20 |
|
| 21 |
class SearchRequest(BaseModel):
|
| 22 |
query: str
|
|
|
|
| 26 |
open_access: bool = False
|
| 27 |
year_from: Optional[str] = None
|
| 28 |
year_to: Optional[str] = None
|
| 29 |
+
resource_type: str = "articles" # "articles" | "books" | "both"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
|
| 31 |
|
| 32 |
+
# ββ PRIMO Boolean β multi-param converter ββββββββββββββββββββββββββββββββββββββ
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
|
| 34 |
+
def _boolean_to_primo_params(boolean_query: str) -> str:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 35 |
"""
|
| 36 |
+
Convert a Boolean query with AND groups + OR alternatives into PRIMO's
|
| 37 |
+
multi-q-parameter format.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
+
PRIMO accepts:
|
| 40 |
+
q=any,contains,smart city OR urban mobility,AND
|
| 41 |
+
&q=any,contains,technocentricity OR citizen-centric
|
| 42 |
|
| 43 |
+
A single q=any,contains,(full boolean string) returns irrelevant results
|
| 44 |
+
because PRIMO treats parentheses and AND/OR as literal search text.
|
| 45 |
|
| 46 |
+
Examples:
|
| 47 |
+
'("smart city" OR "urban mobility") AND ("technocentricity")'
|
| 48 |
+
β 'q=any,contains,smart city OR urban mobility,AND&q=any,contains,technocentricity'
|
|
|
|
| 49 |
|
| 50 |
+
'"climate change"'
|
| 51 |
+
β 'q=any,contains,climate change'
|
| 52 |
+
"""
|
| 53 |
+
if not boolean_query or not boolean_query.strip():
|
| 54 |
+
return f"q=any,contains,{quote('library')}"
|
| 55 |
+
|
| 56 |
+
# If no AND groups, treat as a simple query
|
| 57 |
+
has_and = bool(re.search(r'\bAND\b', boolean_query, re.IGNORECASE))
|
| 58 |
+
has_parens = '(' in boolean_query
|
| 59 |
+
|
| 60 |
+
if not (has_and and has_parens):
|
| 61 |
+
# Simple query: strip quotes and operators, use as-is
|
| 62 |
+
clean = re.sub(r'["\']', '', boolean_query).strip()
|
| 63 |
+
clean = re.sub(r'\b(AND|OR|NOT)\b', ' ', clean, flags=re.IGNORECASE)
|
| 64 |
+
clean = re.sub(r'\s+', ' ', clean).strip()
|
| 65 |
+
return f"q=any,contains,{quote(clean)}"
|
| 66 |
+
|
| 67 |
+
# Parse top-level AND groups by tracking parenthesis depth
|
| 68 |
+
groups = []
|
| 69 |
+
depth = 0
|
| 70 |
+
chunk = ""
|
| 71 |
+
for char in boolean_query:
|
| 72 |
+
if char == '(':
|
| 73 |
+
depth += 1
|
| 74 |
+
chunk += char
|
| 75 |
+
elif char == ')':
|
| 76 |
+
depth -= 1
|
| 77 |
+
chunk += char
|
| 78 |
+
else:
|
| 79 |
+
chunk += char
|
| 80 |
+
# When we close back to depth 0 and the chunk ends with ' AND '
|
| 81 |
+
if depth == 0 and chunk.upper().endswith(' AND '):
|
| 82 |
+
groups.append(chunk[:-5].strip())
|
| 83 |
+
chunk = ""
|
| 84 |
+
if chunk.strip():
|
| 85 |
+
groups.append(chunk.strip())
|
| 86 |
+
|
| 87 |
+
if not groups:
|
| 88 |
+
# Fallback: couldn't parse groups
|
| 89 |
+
clean = re.sub(r'["\']', '', boolean_query).strip()
|
| 90 |
+
return f"q=any,contains,{quote(clean)}"
|
| 91 |
+
|
| 92 |
+
primo_params = []
|
| 93 |
+
for i, group in enumerate(groups):
|
| 94 |
+
g = group.strip()
|
| 95 |
+
# Strip outer parentheses
|
| 96 |
+
if g.startswith('(') and g.endswith(')'):
|
| 97 |
+
g = g[1:-1].strip()
|
| 98 |
+
# Remove quotes β PRIMO handles phrases without them
|
| 99 |
+
g = re.sub(r'["\']', '', g)
|
| 100 |
+
# Normalise OR spacing
|
| 101 |
+
g = re.sub(r'\s+OR\s+', ' OR ', g, flags=re.IGNORECASE).strip()
|
| 102 |
+
if not g:
|
| 103 |
+
continue
|
| 104 |
+
# Add ,AND connector on all groups except the last
|
| 105 |
+
suffix = ',AND' if i < len(groups) - 1 else ''
|
| 106 |
+
primo_params.append(f"q=any,contains,{quote(g)}{suffix}")
|
| 107 |
+
|
| 108 |
+
if not primo_params:
|
| 109 |
+
clean = re.sub(r'["\']', '', boolean_query).strip()
|
| 110 |
+
return f"q=any,contains,{quote(clean)}"
|
| 111 |
+
|
| 112 |
+
return '&'.join(primo_params)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
# ββ PRIMO search βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 116 |
+
|
| 117 |
+
def _build_primo_search_link(title: str, record_id: Optional[str] = None) -> Optional[str]:
|
| 118 |
+
base = 'https://khalifa.primo.exlibrisgroup.com/discovery'
|
| 119 |
+
if record_id:
|
| 120 |
+
return f"{base}/fulldisplay?docid={quote(str(record_id))}&vid=971KUOSTAR_INST:KU"
|
| 121 |
+
clean_title = (title or '').strip()
|
| 122 |
+
if not clean_title:
|
| 123 |
+
return None
|
| 124 |
+
return (
|
| 125 |
+
f"{base}/search?query=any,contains,{quote(clean_title)}"
|
| 126 |
+
"&tab=Everything&search_scope=MyInst_and_CI&vid=971KUOSTAR_INST:KU&lang=en"
|
| 127 |
)
|
| 128 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
|
| 130 |
+
async def search_primo(req: SearchRequest, api_key: str) -> dict:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 131 |
"""
|
| 132 |
+
Search the PRIMO API.
|
| 133 |
+
Converts boolean queries to multi-param format for accurate results.
|
| 134 |
"""
|
| 135 |
+
import httpx
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 136 |
|
| 137 |
+
if not api_key:
|
| 138 |
+
return {"error": "PRIMO_API_KEY not configured", "results": [], "total": 0}
|
| 139 |
|
| 140 |
+
# Decide query format: boolean multi-param vs plain query
|
| 141 |
+
query_str = (req.query or "").strip()
|
| 142 |
+
has_bool = bool(re.search(r'\b(AND|OR)\b', query_str)) and '(' in query_str
|
| 143 |
|
| 144 |
+
if has_bool:
|
| 145 |
+
query_params = _boolean_to_primo_params(query_str)
|
| 146 |
+
else:
|
| 147 |
+
# Plain topic: quote-strip and encode
|
| 148 |
+
clean = re.sub(r'["\']', '', query_str).strip()
|
| 149 |
+
query_params = f"q=any,contains,{quote(clean)}"
|
|
|
|
|
|
|
|
|
|
| 150 |
|
| 151 |
+
# Facets
|
| 152 |
+
facets = ""
|
| 153 |
if req.peer_reviewed:
|
| 154 |
+
facets += "&qInclude=facet_tlevel,exact,peer_reviewed"
|
| 155 |
+
if req.open_access:
|
| 156 |
+
facets += "&qInclude=facet_tlevel,exact,open_access"
|
|
|
|
|
|
|
|
|
|
| 157 |
if req.year_from or req.year_to:
|
| 158 |
yf = req.year_from or "1900"
|
| 159 |
+
yt = req.year_to or str(datetime.now().year)
|
| 160 |
+
facets += f"&multiFacets=facet_searchcreationdate,include,{yf}%7C,%7C{yt}"
|
| 161 |
+
if req.resource_type == "books":
|
| 162 |
+
facets += "&qInclude=facet_rtype,exact,books"
|
| 163 |
+
elif req.resource_type == "articles":
|
| 164 |
+
facets += "&qInclude=facet_rtype,exact,articles"
|
| 165 |
+
|
| 166 |
+
vid = "971KUOSTAR_INST:KU"
|
| 167 |
+
qs = (
|
| 168 |
+
f"?vid={vid}&tab=Everything&scope=MyInst_and_CI"
|
| 169 |
+
f"&{query_params}"
|
| 170 |
+
f"&lang=en&sort=rank&limit={req.limit}&offset=0&mode=advanced"
|
| 171 |
+
f"&apikey={api_key}{facets}"
|
| 172 |
)
|
| 173 |
|
| 174 |
+
base = "https://api-eu.hosted.exlibrisgroup.com/primo/v1/search"
|
| 175 |
+
results = []
|
| 176 |
+
total = 0
|
| 177 |
+
|
| 178 |
+
async with httpx.AsyncClient(timeout=15) as client:
|
| 179 |
+
for region in ["api-eu", "api-na", "api-ap"]:
|
| 180 |
+
url = base.replace("api-eu", region) + qs
|
| 181 |
+
try:
|
| 182 |
+
r = await client.get(url, headers={"Accept": "application/json"})
|
| 183 |
+
if r.status_code == 200:
|
| 184 |
+
data = r.json()
|
| 185 |
+
total = data.get("info", {}).get("total", 0)
|
| 186 |
+
for doc in data.get("docs", []):
|
| 187 |
+
pnx = doc.get("pnx", {})
|
| 188 |
+
d = pnx.get("display", {})
|
| 189 |
+
a = pnx.get("addata", {})
|
| 190 |
+
s = pnx.get("search", {})
|
| 191 |
+
c = pnx.get("control", {})
|
| 192 |
+
l = pnx.get("links", {})
|
| 193 |
+
record_id = (c.get("recordid") or [None])[0]
|
| 194 |
+
title = (d.get("title") or ["Untitled"])[0]
|
| 195 |
+
primo_url = _build_primo_search_link(title, record_id)
|
| 196 |
+
doi = (a.get("doi") or [None])[0]
|
| 197 |
+
raw_link = (l.get("openurl") or l.get("linktorsrc") or [None])[0]
|
| 198 |
+
results.append({
|
| 199 |
+
"record_id": record_id,
|
| 200 |
+
"title": title,
|
| 201 |
+
"creator": "; ".join(
|
| 202 |
+
d.get("creator") or d.get("contributor") or []
|
| 203 |
+
) or "Unknown",
|
| 204 |
+
"date": (
|
| 205 |
+
s.get("creationdate") or a.get("risdate") or a.get("date") or [""]
|
| 206 |
+
)[0],
|
| 207 |
+
"type": (d.get("type") or [""])[0],
|
| 208 |
+
"source": (d.get("source") or a.get("jtitle") or [""])[0],
|
| 209 |
+
"description": ((d.get("description") or [""])[0] or "")[:400],
|
| 210 |
+
"doi": doi,
|
| 211 |
+
"primo_url": primo_url,
|
| 212 |
+
"link": raw_link or primo_url,
|
| 213 |
+
"open_access": (d.get("oa") or [""])[0] == "free_for_read",
|
| 214 |
+
"_source": "PRIMO",
|
| 215 |
+
})
|
| 216 |
+
return {"total": total, "results": results, "source": "PRIMO"}
|
| 217 |
+
except Exception:
|
| 218 |
+
continue
|
| 219 |
+
|
| 220 |
+
return {"error": "PRIMO API unavailable", "results": [], "total": 0, "source": "PRIMO"}
|
| 221 |
+
|
| 222 |
+
|
| 223 |
+
# ββ PubMed search ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 224 |
+
|
| 225 |
+
async def search_pubmed(req: SearchRequest) -> dict:
|
| 226 |
+
"""Search PubMed via E-utilities."""
|
| 227 |
+
import httpx
|
| 228 |
+
|
| 229 |
+
base = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
|
| 230 |
+
query = (req.query or "").strip()
|
| 231 |
+
|
| 232 |
+
# Build date filter if needed
|
| 233 |
+
if req.year_from or req.year_to:
|
| 234 |
+
yf = req.year_from or "1900"
|
| 235 |
+
yt = req.year_to or str(datetime.now().year)
|
| 236 |
+
query = f"({query}) AND ({yf}:{yt}[dp])"
|
| 237 |
+
|
| 238 |
try:
|
| 239 |
async with httpx.AsyncClient(timeout=15) as client:
|
| 240 |
+
r = await client.get(
|
| 241 |
+
f"{base}/esearch.fcgi",
|
| 242 |
+
params={
|
| 243 |
+
"db": "pubmed", "term": query,
|
| 244 |
+
"retmax": req.limit, "retmode": "json", "sort": "relevance"
|
| 245 |
+
}
|
| 246 |
)
|
|
|
|
| 247 |
if r.status_code != 200:
|
|
|
|
| 248 |
return {"error": "PubMed search failed", "results": [], "source": "PubMed"}
|
| 249 |
|
| 250 |
ids = r.json().get("esearchresult", {}).get("idlist", [])
|
| 251 |
+
count = int(r.json().get("esearchresult", {}).get("count", 0))
|
| 252 |
if not ids:
|
| 253 |
return {"total": 0, "results": [], "source": "PubMed"}
|
| 254 |
|
| 255 |
+
r2 = await client.get(
|
| 256 |
+
f"{base}/esummary.fcgi",
|
| 257 |
+
params={"db": "pubmed", "id": ",".join(ids), "retmode": "json"}
|
| 258 |
)
|
|
|
|
| 259 |
if r2.status_code != 200:
|
|
|
|
| 260 |
return {"error": "PubMed fetch failed", "results": [], "source": "PubMed"}
|
| 261 |
|
| 262 |
data = r2.json().get("result", {})
|
|
|
|
| 265 |
rec = data.get(pid, {})
|
| 266 |
if not isinstance(rec, dict):
|
| 267 |
continue
|
| 268 |
+
authors_list = rec.get("authors", [])
|
| 269 |
+
authors = ", ".join(a.get("name", "") for a in authors_list[:3])
|
| 270 |
+
if len(authors_list) > 3:
|
| 271 |
authors += " et al."
|
| 272 |
+
eloc = rec.get("elocationid", "")
|
| 273 |
+
doi = eloc.replace("doi: ", "") if "doi:" in eloc else None
|
| 274 |
results.append({
|
| 275 |
"title": rec.get("title", ""),
|
| 276 |
"creator": authors,
|
| 277 |
"date": rec.get("pubdate", ""),
|
| 278 |
"source": rec.get("fulljournalname", rec.get("source", "")),
|
| 279 |
+
"doi": doi,
|
| 280 |
"pmid": pid,
|
| 281 |
"link": f"https://pubmed.ncbi.nlm.nih.gov/{pid}/",
|
| 282 |
"type": "Journal Article",
|
| 283 |
"_source": "PubMed",
|
| 284 |
})
|
| 285 |
+
return {"total": count, "results": results, "source": "PubMed"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 286 |
|
| 287 |
+
except Exception as e:
|
| 288 |
+
return {"error": f"PubMed: {str(e)}", "results": [], "source": "PubMed"}
|
|
|
|
|
|