Upload 7 files

README.md

@@ -29,7 +29,9 @@ Turns a natural-language question into keyword searches on **Khazna** (KU's Else
 
 **Concept expansion:** for topic, thesis, expert, project and collaboration searches, one Pure thesaurus concept from `/concepts/search` that shares a word with the query is added as an extra search variant. Set `USE_CONCEPTS=false` to switch it off.
 
-**Current KU staff only:** `/persons` never includes external persons. By default the bot also drops anyone without a current staff appointment, meaning a `staffOrganizationAssociations` entry with no `period.endDate` or one in the future. That leaves out former staff and student-only records. If the API returns appointment data for some people in a result set, anyone without it is treated as not current. If it returns none at all, people are kept. When a named researcher has no current KU profile, the bot only returns publications that have a **KU** author of that name, never ones that match an external co-author. `GET /debug/persons?q=Kumar` (admin) lists every person Pure returned, their appointments, and why each was shown or dropped. Set `PURE_ACTIVE_PERSONS_ONLY=false` to list everyone. A researcher the user explicitly picks is always shown. Khazna people-search links add `community=false`, so external Pure Community profiles are excluded.
+**Who is listed comes from the Khazna portal:** for a named researcher, the bot reads the portal's own "Find Profiles" search (`/en/persons/?search=<name>…&community=false`). That gives current, public, KU-only profiles, the same list people see on Khazna. The API adds each person's uuid, role and unit. Someone the portal lists but the API can't match is still shown and explored by name (`person_name`). If the portal can't be reached, the bot falls back to `/persons/search`. Set `USE_PORTAL_PEOPLE=false` to always use the API. Tests use a real saved portal page (`tests/fixtures/portal_persons_kumar.html`, 8 Kumars).
+
+**Current KU staff only (API fallback):** `/persons` never includes external persons. By default the bot also drops anyone without a current staff appointment, meaning a `staffOrganizationAssociations` entry with no `period.endDate` or one in the future. That leaves out former staff and student-only records. KU's Pure does not return appointment data for every current staff member, so a person with no appointment data is kept. Only an ended appointment drops someone. When a named researcher has no current KU profile, the bot only returns publications that have a **KU** author of that name, never ones that match an external co-author. `GET /debug/persons?q=Kumar` (admin) lists every person Pure returned, their appointments, and why each was shown or dropped. Set `PURE_ACTIVE_PERSONS_ONLY=false` to list everyone. A researcher the user explicitly picks is always shown. Khazna people-search links add `community=false`, so external Pure Community profiles are excluded.
 
 **Name disambiguation:** for a named researcher, `/persons/search` returns up to 50 people, kept only if every typed name word equals one of their name words (first, middle or last; prefix matches only when nothing matches exactly). One match, or one exact full-name match, goes straight to that profile. Two or more return `intent: "author_clarify"` with up to `MAX_NAME_CANDIDATES` (default 12) cards marked `choose: true`. The frontend then resends the original question with `person_uuid`, which bypasses the name search entirely.
 

app.py

@@ -146,6 +146,7 @@ class AgentRequest(BaseModel):
     top_k: int | None = None
     scope: str = "all"            # all | publications | theses | persons | organisations | projects | equipment | collaborations
     person_uuid: str | None = None  # set when the user picks one researcher from a disambiguation list
+    person_name: str | None = None  # same, for a portal-listed researcher without an API uuid
 
 
 class FeedbackRequest(BaseModel):
@@ -201,8 +202,7 @@ def _reply(t0: float, **kw) -> dict:
 _SESSION_RE = re.compile(r"^[A-Za-z0-9_-]{8,80}$")
 
 
-PERSONS_PORTAL_PARAMS = ("&pageSize=50&ordering=rating&descending=false&showAdvanced=false&allConcepts=true"
-                         "&inferConcepts=true&searchBy=RelatedConcepts&community=false&direct2experts=false")
+from pure_client import PERSONS_PORTAL_PARAMS
 
 
 def _portal_url(path: str, term: str, search_by: str | None = None) -> str:
@@ -341,7 +341,8 @@ async def agent(req: AgentRequest, request: Request, response: Response):
 
     person_uuid = req.person_uuid if req.person_uuid and re.fullmatch(r"[A-Za-z0-9-]{1,64}", req.person_uuid) else None
     try:
-        retrieval = await pipeline.retrieve(pure, plan, person_uuid=person_uuid)
+        person_name = (req.person_name or "").strip()[:120] or None
+        retrieval = await pipeline.retrieve(pure, plan, person_uuid=person_uuid, person_name=person_name)
     except PureError as e:
         log_query(q, plan.intent, plan.keyword_queries, 0, model, time.time() - t0, str(e), **logx)
         if e.status == 403 and plan.intent == "dataset_search":
@@ -376,7 +377,12 @@ async def agent(req: AgentRequest, request: Request, response: Response):
         log_query(q, "author_not_found", [name], 0, model, time.time() - t0, **logx)
         return _reply(t0, **base, answer=answer, trace=[{k: v for k, v in t.items() if k != "error"} for t in retrieval["trace"]],
                       portal_search_url=_portal_url("persons", name), follow_up_suggestions=[])
-    if not results:
+    if not results and retrieval.get("person"):
+        pname = retrieval["person"].get("name", "this researcher")
+        answer = (f"Here is **{pname}**'s Khazna profile. I couldn't find matching publications through search"
+                  + (" for that topic or filter" if (plan.keyword_queries and plan.intent == "author_search") else "")
+                  + "; open the profile for their full list of outputs.")
+    elif not results:
         filt = bool(plan.year_from or plan.year_to or plan.output_types or plan.open_access_only)
         answer = ("I couldn't find Khazna records matching that" + (" with the filters you asked for" if filt else "")
                   + ". Try broader or different keywords" + (", or remove the year/type filter." if filt else "."))
@@ -478,15 +484,20 @@ async def debug_persons(q: str = "Kumar", x_admin_password: str | None = Header(
                 "current": pure_client._association_active(a, today)})
         reason = ("not public" if not pure_client.is_public(i) else
                   "no current staff appointment" if status is False else
-                  "no staff appointments returned" if status is None and field_returned else
-                  "name does not match" if i.get("uuid") not in matched_uuids else "shown")
+                  "name does not match" if i.get("uuid") not in matched_uuids else "kept")
         rows.append({"name": pure_client._person_name(i.get("name")), "uuid": i.get("uuid"),
                      "visibility": vis.get("key") if isinstance(vis, dict) else vis,
                      "staff_appointments": assocs,
                      "other_association_fields": [k for k in i if k.endswith("Associations") and k != "staffOrganizationAssociations" and i.get(k)],
                      "kept_by_filter": i.get("uuid") in kept_uuids, "result": reason})
-    return {"query": q, "pure_count": raw.get("count"), "returned": len(items), "staff_field_returned": field_returned,
-            "shown": sum(r["result"] == "shown" for r in rows), "people": rows}
+    portal = await pure.portal_person_search(q)
+    merged = pipeline.merge_portal_people(portal or [], await pure.search_persons(q, size=50)) if portal is not None else None
+    shown_by_bot, _ = pipeline.match_people(q, merged if merged is not None else [])
+    return {"query": q, "portal_reachable": portal is not None,
+            "portal_people": [{"name": p["name"], "portal_url": p["portal_url"], "api_uuid": p.get("uuid") or None,
+                               "shown_by_bot": p in shown_by_bot} for p in (merged or [])],
+            "api_count": raw.get("count"), "api_returned": len(items), "staff_field_returned": field_returned,
+            "api_people": rows}
 
 
 @app.get("/debug/pure")

pipeline.py

@@ -336,17 +336,63 @@ def match_people(query: str, people: list[dict]) -> tuple[list[dict], list[dict]
     return sorted(matches, key=order), full
 
 
-async def _resolve_person(pure: PureClient, plan: QueryPlan, person_uuid: str | None, trace: list[dict]):
-    """→ (person, candidates). candidates is non-empty only when the name is ambiguous."""
+USE_PORTAL_PEOPLE = os.environ.get("USE_PORTAL_PEOPLE", "true").lower() != "false"
+
+
+def _portal_key(url: str) -> str:
+    return (url or "").split("?")[0].rstrip("/").lower()
+
+
+def merge_portal_people(portal: list[dict], api_people: list[dict]) -> list[dict]:
+    """The portal decides WHO is listed (current KU profiles); the API adds uuid, role, unit."""
+    by_url = {_portal_key(p.get("portal_url", "")): p for p in api_people if p.get("portal_url")}
+    by_name: dict[tuple, list[dict]] = {}
+    for p in api_people:
+        by_name.setdefault(tuple(sorted(_name_tokens(p.get("name", "")))), []).append(p)
+    merged = []
+    for entry in portal:
+        api = by_url.get(_portal_key(entry["portal_url"]))
+        if api is None:
+            same = by_name.get(tuple(sorted(_name_tokens(entry["name"]))), [])
+            api = same[0] if len(same) == 1 else None
+        if api:
+            merged.append({**api, "name": entry["name"], "portal_url": entry["portal_url"]})
+        else:
+            parts = entry["name"].split()
+            merged.append({"kind": "person", "uuid": "", "name": entry["name"], "title": entry["name"],
+                           "first_name": " ".join(parts[:-1]), "last_name": parts[-1] if parts else "",
+                           "job_titles": [], "organisations": [], "keywords": [], "orcid": "",
+                           "active": True, "portal_url": entry["portal_url"]})
+    return merged
+
+
+async def _resolve_person(pure: PureClient, plan: QueryPlan, person_uuid: str | None, trace: list[dict],
+                          exact_name: str | None = None):
+    """→ (person, candidates). candidates is non-empty only when the name is ambiguous.
+
+    Who counts as a KU researcher comes from the Khazna portal's own profile search
+    (current, public, KU-only). The API search is used to add details, and on its own
+    only if the portal can't be reached.
+    """
     if person_uuid:
         person = await pure.get_person(person_uuid, require_active=False)
         trace.append({"endpoint": f"persons/{person_uuid}", "hits": 1 if person else 0})
         return person, []
-    people = await pure.search_persons(plan.author_name, size=50)
-    trace.append({"endpoint": "persons/search", "query": plan.author_name, "hits": len(people)})
-    matches, full = match_people(plan.author_name, people)
-    if not matches and len(people) == 1:
-        matches = people
+    name = exact_name or plan.author_name
+    portal = await pure.portal_person_search(name) if USE_PORTAL_PEOPLE else None
+    api_people = await pure.search_persons(name, size=50)
+    trace.append({"endpoint": "persons/search", "query": name, "hits": len(api_people)})
+    if portal is not None:
+        trace.append({"endpoint": "portal/persons", "hits": len(portal)})
+        pool = merge_portal_people(portal, api_people)
+    else:
+        pool = api_people
+    matches, full = match_people(name, pool)
+    if exact_name:  # the user picked this person from a list: never ask again
+        pick = full or matches
+        return (pick[0] if pick else None), []
+    if not matches and len(pool) == 1:
+        matches = pool
     if len(full) == 1:
         return full[0], []
     if len(matches) == 1:
@@ -356,22 +402,23 @@ async def _resolve_person(pure: PureClient, plan: QueryPlan, person_uuid: str |
     return None, []
 
 
-async def _retrieve_outputs(pure: PureClient, plan: QueryPlan, search=None, person_uuid: str | None = None) -> dict:
+async def _retrieve_outputs(pure: PureClient, plan: QueryPlan, search=None, person_uuid: str | None = None,
+                            exact_name: str | None = None) -> dict:
     search = search or pure.search_research_outputs
     trace: list[dict] = []
     person = None
     total = 0
     name_filter = None  # set when a named researcher has no current KU profile
 
-    if plan.intent == "author_search" and (plan.author_name or person_uuid):
-        person, candidates = await _resolve_person(pure, plan, person_uuid, trace)
+    if plan.intent == "author_search" and (plan.author_name or person_uuid or exact_name):
+        person, candidates = await _resolve_person(pure, plan, person_uuid, trace, exact_name=exact_name)
         if candidates:  # ambiguous name: ask the user to choose instead of guessing
             shown = candidates[:MAX_CANDIDATES]
             await pure.fill_org_names(shown)
             return {"results": [{**c, "choose": True} for c in shown], "unfiltered": len(candidates),
                     "total": len(candidates), "person": None, "trace": trace,
                     "clarify": {"name": plan.author_name, "count": len(candidates), "shown": len(shown)}}
-        if person:
+        if person and person.get("uuid"):
             await pure.fill_org_names([person])
             outs = await pure.person_research_outputs(person, size=100)
             trace.append({"endpoint": f"persons/{person['uuid']}/research-outputs", "hits": len(outs)})
@@ -381,7 +428,9 @@ async def _retrieve_outputs(pure: PureClient, plan: QueryPlan, search=None, pers
                 outs = topical or outs
             variant_hits = [outs]
             total = len(outs)
-        else:  # no current KU profile: search outputs, but keep only those with a KU author of that name
+        else:  # no API record (or none at all): search outputs, keep only those with a KU author of that name
+            if person:  # listed on the portal but not matched to an API record: search by their full name
+                plan.author_name = person["name"]
             name_filter = plan.author_name
             plan.keyword_queries = [f"{plan.author_name or ''} {q}".strip() for q in (plan.keyword_queries or [""])]
             plan.keyword_queries = [q for q in plan.keyword_queries if q] or [plan.standalone_question]
@@ -417,7 +466,7 @@ async def _retrieve_outputs(pure: PureClient, plan: QueryPlan, search=None, pers
         qt = _name_tokens(name_filter)
         result["results"] = [o for o in filtered
                              if any(qt and all(t in _name_tokens(a.get("name", "")) for t in qt) for a in o.get("ku_authors", []))]
-        if not result["results"]:
+        if not result["results"] and person is None:
             result["no_person"] = name_filter
     return result
 
@@ -591,10 +640,13 @@ async def _retrieve_datasets(pure: PureClient, plan: QueryPlan) -> dict:
     return r
 
 
-async def retrieve(pure: PureClient, plan: QueryPlan, person_uuid: str | None = None) -> dict:
-    if person_uuid:  # the user picked a specific researcher from a list
+async def retrieve(pure: PureClient, plan: QueryPlan, person_uuid: str | None = None,
+                   person_name: str | None = None) -> dict:
+    if person_uuid or person_name:  # the user picked a specific researcher from a list
         plan.intent = "author_search"
-        return await _retrieve_outputs(pure, plan, person_uuid=person_uuid)
+        if person_name:
+            plan.author_name = person_name
+        return await _retrieve_outputs(pure, plan, person_uuid=person_uuid, exact_name=None if person_uuid else person_name)
     concept = None
     if plan.intent in ("topic_search", "thesis_search", "expert_search", "project_search", "collab_search", "count"):
         concept = await _expand_with_concepts(pure, plan)

pure_client.py

@@ -29,6 +29,9 @@ import re
 import time
 from typing import Any
 
+import html as _html
+from urllib.parse import quote_plus
+
 import httpx
 
 # Localised-string keys we prefer, in order. Pure portals are usually en_GB.
@@ -500,6 +503,27 @@ def normalize_external_organization(item: dict, portal_url: str) -> dict:
     }
 
 
+# Same parameters as the portal's own "Find Profiles" page: community=false leaves out
+# external Pure Community profiles, so only KU's current public profiles are listed.
+PERSONS_PORTAL_PARAMS = ("&pageSize=50&ordering=rating&descending=false&showAdvanced=false&allConcepts=true"
+                         "&inferConcepts=true&searchBy=RelatedConcepts&community=false&direct2experts=false")
+_PORTAL_PERSON_RE = re.compile(
+    r'<a\b(?=[^>]*\bclass="[^"]*\blink person\b[^"]*")(?=[^>]*\bhref="([^"]+)")[^>]*>\s*<span>(.*?)</span>', re.S)
+
+
+def parse_portal_persons(page: str, portal_url: str) -> list[dict]:
+    """Names and profile links from a Khazna portal 'Find Profiles' result page."""
+    out, seen = [], set()
+    for href, raw in _PORTAL_PERSON_RE.findall(page):
+        name = _clean_text(_html.unescape(raw))
+        url = href if href.startswith("http") else portal_url.rstrip("/") + href
+        key = url.rstrip("/").lower()
+        if name and key not in seen:
+            seen.add(key)
+            out.append({"name": name, "portal_url": url})
+    return out
+
+
 # ───────────────────────────── client ───────────────────────────────
 class PureClient:
     def __init__(
@@ -525,9 +549,31 @@ class PureClient:
             headers={"api-key": api_key, "Accept": "application/json", "Content-Type": "application/json"},
             transport=transport,
         )
+        # Separate client for the public portal pages: never sends the API key.
+        self._portal = httpx.AsyncClient(timeout=15.0, transport=transport, follow_redirects=True,
+                                         headers={"Accept": "text/html", "User-Agent": "KU-Khazna-AI-Assistant/1.0"})
 
     async def aclose(self):
         await self._client.aclose()
+        await self._portal.aclose()
+
+    async def portal_person_search(self, name: str) -> list[dict] | None:
+        """KU people the Khazna portal lists for `name` (current, public, no external profiles).
+        Returns None if the portal can't be reached, so callers can fall back to the API."""
+        key = f"portal-persons|{name.lower()}"
+        hit = self._cache.get(key)
+        if hit and time.time() - hit[0] < self.cache_ttl:
+            return hit[1]
+        url = f"{self.portal_url}/en/persons/?search={quote_plus(name)}{PERSONS_PORTAL_PARAMS}"
+        try:
+            r = await self._portal.get(url)
+        except (httpx.TimeoutException, httpx.TransportError):
+            return None
+        if r.status_code != 200:
+            return None
+        people = parse_portal_persons(r.text, self.portal_url)
+        self._cache[key] = (time.time(), people)
+        return people
 
     # -- low level --------------------------------------------------
     async def request(self, method: str, path: str, json: dict | None = None, params: dict | None = None) -> dict:
@@ -587,15 +633,9 @@ class PureClient:
         items = self._filter(data.get("items") or [])
         if not self.active_persons_only:
             return [normalize_person(i, self.portal_url) for i in items]
-        # If the API returns staff associations for anyone in this batch, a person without
-        # them is not current staff (rather than "unknown"), so they are dropped too.
-        field_returned = any("staffOrganizationAssociations" in i for i in items)
-        kept = []
-        for i in items:
-            status = staff_status(i)
-            if status is True or (status is None and not field_returned):
-                kept.append(normalize_person(i, self.portal_url))
-        return kept
+        # KU's Pure does not return staff appointments for every person, so missing data
+        # means "unknown" and the person is kept; only an ended appointment drops someone.
+        return [normalize_person(i, self.portal_url) for i in items if staff_status(i) is not False]
 
     async def person_research_outputs(self, person: dict, size: int = 50) -> list[dict]:
         """Outputs where `person` is a contributor.