KevinIsInCoding Claude Sonnet 4.6 commited on
Commit
45900d6
·
1 Parent(s): 5fb461d

fix(pmc): use per-PMID elink instead of batch to preserve ID mapping

Browse files

Batch elink merges all source IDs into one LinkSet — no way to map
which PMC ID belongs to which PMID. Send one at a time instead.
Also removes defunct NCBI ID Converter API (returning 404).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Files changed (1) hide show
  1. ingestion/pmc.py +11 -10
ingestion/pmc.py CHANGED
@@ -23,12 +23,15 @@ def _configure_entrez() -> None:
23
 
24
 
25
  def _sleep() -> None:
 
26
  time.sleep(0.1 if os.getenv("NCBI_API_KEY") else 0.4)
27
 
28
 
29
  def get_pmcids(pmids: list[str]) -> dict[str, str]:
30
  """
31
  Map PubMed IDs to PMC IDs for papers with Open Access full text.
 
 
32
  Returns {pmid: pmcid}.
33
  """
34
  _configure_entrez()
@@ -36,30 +39,28 @@ def get_pmcids(pmids: list[str]) -> dict[str, str]:
36
  return {}
37
 
38
  result: dict[str, str] = {}
39
- for i in range(0, len(pmids), 200):
40
- batch = pmids[i : i + 200]
41
  for attempt in range(3):
42
  try:
43
- handle = Entrez.elink(dbfrom="pubmed", db="pmc", id=",".join(batch))
44
  link_sets = Entrez.read(handle)
45
  handle.close()
46
  break
47
  except Exception as exc:
48
  if attempt == 2:
49
- _logger.warning(f"elink failed: {exc}")
50
  link_sets = []
51
  break
52
  time.sleep(2 ** attempt)
53
 
54
- for link_set in link_sets:
55
- source_ids = link_set.get("IdList", [])
56
- source_id = str(source_ids[0]) if source_ids else None
57
- for db_link in link_set.get("LinkSetDb", []):
58
  if db_link.get("DbTo") == "pmc":
59
  links = db_link.get("Link", [])
60
- if links and source_id:
61
- result[source_id] = str(links[0]["Id"])
62
  break
 
63
  _sleep()
64
 
65
  _logger.info("PMC ID lookup", extra={"data": {"pmids": len(pmids), "found": len(result)}})
 
23
 
24
 
25
  def _sleep() -> None:
26
+ # NCBI rate limit: 10 req/s with API key, 3 req/s without
27
  time.sleep(0.1 if os.getenv("NCBI_API_KEY") else 0.4)
28
 
29
 
30
  def get_pmcids(pmids: list[str]) -> dict[str, str]:
31
  """
32
  Map PubMed IDs to PMC IDs for papers with Open Access full text.
33
+ Sends one PMID at a time to elink — batch elink merges all results
34
+ into one LinkSet with no per-ID mapping, making it unusable for this purpose.
35
  Returns {pmid: pmcid}.
36
  """
37
  _configure_entrez()
 
39
  return {}
40
 
41
  result: dict[str, str] = {}
42
+ for pmid in pmids:
 
43
  for attempt in range(3):
44
  try:
45
+ handle = Entrez.elink(dbfrom="pubmed", db="pmc", id=pmid)
46
  link_sets = Entrez.read(handle)
47
  handle.close()
48
  break
49
  except Exception as exc:
50
  if attempt == 2:
51
+ _logger.debug(f"elink failed for PMID {pmid}: {exc}")
52
  link_sets = []
53
  break
54
  time.sleep(2 ** attempt)
55
 
56
+ for ls in link_sets:
57
+ for db_link in ls.get("LinkSetDb", []):
 
 
58
  if db_link.get("DbTo") == "pmc":
59
  links = db_link.get("Link", [])
60
+ if links:
61
+ result[pmid] = str(links[0]["Id"])
62
  break
63
+
64
  _sleep()
65
 
66
  _logger.info("PMC ID lookup", extra={"data": {"pmids": len(pmids), "found": len(result)}})