Spaces:
Sleeping
Sleeping
KevinIsInCoding Claude Sonnet 4.6 commited on
Commit ·
45900d6
1
Parent(s): 5fb461d
fix(pmc): use per-PMID elink instead of batch to preserve ID mapping
Browse filesBatch elink merges all source IDs into one LinkSet — no way to map
which PMC ID belongs to which PMID. Send one at a time instead.
Also removes defunct NCBI ID Converter API (returning 404).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
- ingestion/pmc.py +11 -10
ingestion/pmc.py
CHANGED
|
@@ -23,12 +23,15 @@ def _configure_entrez() -> None:
|
|
| 23 |
|
| 24 |
|
| 25 |
def _sleep() -> None:
|
|
|
|
| 26 |
time.sleep(0.1 if os.getenv("NCBI_API_KEY") else 0.4)
|
| 27 |
|
| 28 |
|
| 29 |
def get_pmcids(pmids: list[str]) -> dict[str, str]:
|
| 30 |
"""
|
| 31 |
Map PubMed IDs to PMC IDs for papers with Open Access full text.
|
|
|
|
|
|
|
| 32 |
Returns {pmid: pmcid}.
|
| 33 |
"""
|
| 34 |
_configure_entrez()
|
|
@@ -36,30 +39,28 @@ def get_pmcids(pmids: list[str]) -> dict[str, str]:
|
|
| 36 |
return {}
|
| 37 |
|
| 38 |
result: dict[str, str] = {}
|
| 39 |
-
for
|
| 40 |
-
batch = pmids[i : i + 200]
|
| 41 |
for attempt in range(3):
|
| 42 |
try:
|
| 43 |
-
handle = Entrez.elink(dbfrom="pubmed", db="pmc", id=
|
| 44 |
link_sets = Entrez.read(handle)
|
| 45 |
handle.close()
|
| 46 |
break
|
| 47 |
except Exception as exc:
|
| 48 |
if attempt == 2:
|
| 49 |
-
_logger.
|
| 50 |
link_sets = []
|
| 51 |
break
|
| 52 |
time.sleep(2 ** attempt)
|
| 53 |
|
| 54 |
-
for
|
| 55 |
-
|
| 56 |
-
source_id = str(source_ids[0]) if source_ids else None
|
| 57 |
-
for db_link in link_set.get("LinkSetDb", []):
|
| 58 |
if db_link.get("DbTo") == "pmc":
|
| 59 |
links = db_link.get("Link", [])
|
| 60 |
-
if links
|
| 61 |
-
result[
|
| 62 |
break
|
|
|
|
| 63 |
_sleep()
|
| 64 |
|
| 65 |
_logger.info("PMC ID lookup", extra={"data": {"pmids": len(pmids), "found": len(result)}})
|
|
|
|
| 23 |
|
| 24 |
|
| 25 |
def _sleep() -> None:
|
| 26 |
+
# NCBI rate limit: 10 req/s with API key, 3 req/s without
|
| 27 |
time.sleep(0.1 if os.getenv("NCBI_API_KEY") else 0.4)
|
| 28 |
|
| 29 |
|
| 30 |
def get_pmcids(pmids: list[str]) -> dict[str, str]:
|
| 31 |
"""
|
| 32 |
Map PubMed IDs to PMC IDs for papers with Open Access full text.
|
| 33 |
+
Sends one PMID at a time to elink — batch elink merges all results
|
| 34 |
+
into one LinkSet with no per-ID mapping, making it unusable for this purpose.
|
| 35 |
Returns {pmid: pmcid}.
|
| 36 |
"""
|
| 37 |
_configure_entrez()
|
|
|
|
| 39 |
return {}
|
| 40 |
|
| 41 |
result: dict[str, str] = {}
|
| 42 |
+
for pmid in pmids:
|
|
|
|
| 43 |
for attempt in range(3):
|
| 44 |
try:
|
| 45 |
+
handle = Entrez.elink(dbfrom="pubmed", db="pmc", id=pmid)
|
| 46 |
link_sets = Entrez.read(handle)
|
| 47 |
handle.close()
|
| 48 |
break
|
| 49 |
except Exception as exc:
|
| 50 |
if attempt == 2:
|
| 51 |
+
_logger.debug(f"elink failed for PMID {pmid}: {exc}")
|
| 52 |
link_sets = []
|
| 53 |
break
|
| 54 |
time.sleep(2 ** attempt)
|
| 55 |
|
| 56 |
+
for ls in link_sets:
|
| 57 |
+
for db_link in ls.get("LinkSetDb", []):
|
|
|
|
|
|
|
| 58 |
if db_link.get("DbTo") == "pmc":
|
| 59 |
links = db_link.get("Link", [])
|
| 60 |
+
if links:
|
| 61 |
+
result[pmid] = str(links[0]["Id"])
|
| 62 |
break
|
| 63 |
+
|
| 64 |
_sleep()
|
| 65 |
|
| 66 |
_logger.info("PMC ID lookup", extra={"data": {"pmids": len(pmids), "found": len(result)}})
|