Spaces:
Sleeping
Sleeping
File size: 5,037 Bytes
b92b744 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 | """Versioned, inspectable corpus eligibility; no year/type/abstract restriction."""
import json
import re
from app.ingest.openalex import reconstruct_abstract
SCOPE_VERSION = "hypertension-kidney-v1"
PROFILES = ("hypertension-kidney", "hypertension-kidney-pubmed", "all")
# The -pubmed profile is the same topical scope restricted to PubMed-indexed works.
# Update-date partitions differ enormously in quality: some hold almost no PMIDs or
# DOIs and many records whose title and abstract belong to different works. Registry
# linking and Europe PMC full text both need the PubMed identity, so a corpus meant
# for trial evidence can require it. It is opt-in and recorded as its own scope.
PUBMED_SUFFIX = "-pubmed"
TOPIC_IDS = {
"T10144",
"T11839",
"T10728",
"T12910",
"T11001",
"T10373",
"T10449",
"T10606",
"T11174",
}
WORD_PATTERN = (
r"\b(?:hypertens\w*|kidney\w*|renal|renovascular|nephro\w*|glomerul\w*|"
r"dialys\w*|hemodial\w*|haemodial\w*|uremi\w*|uraemi\w*|"
r"albuminuri\w*|proteinuri\w*|podocyt\w*|ureter\w*|urolith\w*|"
r"CKD|ESRD|ESKD|AKI|ADPKD|ARPKD)\b"
)
WORDS = re.compile(WORD_PATTERN, re.IGNORECASE)
PHRASES = re.compile(r"\b(?:high|elevated)\s+blood\s+pressure\b", re.IGNORECASE)
CTGOV_QUERY = (
'AREA[ConditionSearch](hypertension OR "high blood pressure" OR kidney OR renal OR '
"nephrology OR nephropathy OR glomerulonephritis OR dialysis OR hemodialysis OR "
'haemodialysis OR "chronic kidney disease" OR "acute kidney injury" OR '
'"end stage renal disease" OR proteinuria OR albuminuria OR nephrolithiasis)'
)
def is_pubmed_indexed(work: dict) -> bool:
ids = work.get("ids")
if isinstance(ids, str):
try:
ids = json.loads(ids)
except json.JSONDecodeError:
ids = {}
indexed = work.get("indexed_in")
return bool(
(isinstance(ids, dict) and ids.get("pmid"))
or work.get("pmid")
or (isinstance(indexed, (list, tuple)) and "pubmed" in indexed)
)
def matches_work(work: dict, profile: str = "hypertension-kidney", topic: str = "") -> bool:
if profile not in PROFILES:
raise ValueError(f"Unknown scope: {profile}")
if profile.endswith(PUBMED_SUFFIX):
return is_pubmed_indexed(work) and matches_work(
work, profile.removesuffix(PUBMED_SUFFIX), topic
)
topics = work.get("topics") or []
topic_text = json.dumps(topics, ensure_ascii=False).lower()
if topic and topic.lower() not in topic_text:
return False
if profile == "all":
return True
if isinstance(topics, list):
for item in topics:
if not isinstance(item, dict):
continue
if str(item.get("id", "")).rsplit("/", 1)[-1] in TOPIC_IDS:
return True
subfield = item.get("subfield") or {}
if (
isinstance(subfield, dict)
and str(subfield.get("id", "")).rsplit("/", 1)[-1] == "2727"
):
return True
text = " ".join(
[
str(work.get("title") or work.get("display_name") or ""),
reconstruct_abstract(work.get("abstract_inverted_index")),
topic_text,
]
)
return bool(WORDS.search(text) or PHRASES.search(text))
def coarse_predicate(columns: set[str], profile: str) -> str:
"""Cheap SQL prefilter; exact phrase matching follows abstract reconstruction."""
if profile == "all":
return "true"
if profile.endswith(PUBMED_SUFFIX):
# Text casts keep this valid whether ids is a MAP, a STRUCT or a JSON string.
identity = [
f"lower(CAST(\"{name}\" AS VARCHAR)) LIKE '%{needle}%'"
for name, needle in (("ids", "pmid"), ("indexed_in", "pubmed"))
if name in columns
]
if "pmid" in columns:
identity.append('"pmid" IS NOT NULL')
topical = coarse_predicate(columns, profile.removesuffix(PUBMED_SUFFIX))
return f"({' OR '.join(identity) or 'false'}) AND {topical}"
fields = [
f"coalesce(CAST(\"{name}\" AS VARCHAR), '')"
for name in ("title", "display_name", "abstract_inverted_index", "topics")
if name in columns
]
text = "lower(" + " || ' ' || ".join(fields) + ")"
# Inverted indexes do not preserve phrase order. Retain pressure candidates
# here, then match PHRASES against the reconstructed abstract.
patterns = [
"hypertens",
"kidney",
"renal",
"nephro",
"glomerul",
"dialys",
"hemodial",
"haemodial",
"uremi",
"uraemi",
"albuminuri",
"proteinuri",
"podocyt",
"ureter",
"urolith",
"ckd",
"esrd",
"eskd",
"aki",
"adpkd",
"arpkd",
"pressure",
"2727",
*sorted(identifier.lower() for identifier in TOPIC_IDS),
]
return f"regexp_matches({text}, '{'|'.join(patterns)}')"
|