""" Backfill missing DOIs on existing Special Collections items via fuzzy title+author matching against OpenAlex. Some sources (AJOL HTML scrapes especially, and OAI-PMH records with incomplete dc:identifier metadata) yield items with no DOI at all. This script looks each one up on OpenAlex by title, and — mirroring uraas/spiders/sources/isni_spider.py's "exactly one confident candidate or skip" philosophy — only accepts a match when it's unambiguous. It never guesses: a wrong DOI attached to the wrong paper is worse than a missing one. On accept, backfills doi / openalex_id / cited_by_count / counts_by_year. Does NOT touch item_affiliations — run scripts/backfill_collaboration_data.py afterward for that (it's idempotent/skip-if-already-populated, so the two scripts compose naturally as sequential passes). Funder/grant enrichment is deliberately out of scope here — it needs a new Item.funders column (not yet added) and is much simpler once a DOI exists (direct OpenAlex work lookup by DOI, no fuzzy matching needed at all), so it belongs in a follow-up script that runs after this one. Usage: python scripts/backfill_openalex_doi_match.py # DRY RUN python scripts/backfill_openalex_doi_match.py --apply python scripts/backfill_openalex_doi_match.py --apply --limit 50 """ import argparse import json import os import sys import time sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from thefuzz import fuzz from uraas.database import Item, SessionLocal from uraas.utils.analytics_cache import analytics_cache from uraas.utils.openalex_client import oa_get TITLE_THRESHOLD = 92 # near-unique signal — strict on purpose AUTHOR_THRESHOLD = 80 # corroborating only — loose to tolerate name-order/transliteration variance CANDIDATES_PER_QUERY = 5 SELECT = "id,doi,title,authorships,cited_by_count,counts_by_year" def _first_author_name(item) -> str: """Best-effort 'first' author. item_authors has no sequence/order column, so this is a heuristic (usually reflects insertion order), used only as a corroborating signal — never the primary accept/reject gate.""" authors = list(item.authors) return authors[0].name if authors and authors[0].name else "" def _candidate_author_names(work) -> list: return [ (a.get("author", {}) or {}).get("display_name", "") for a in work.get("authorships", []) if (a.get("author", {}) or {}).get("display_name") ] def find_match(title: str, local_author: str): """Return (work_or_None, title_sim, num_candidates_seen, num_qualifying).""" data = oa_get( "/works", {"search": title, "select": SELECT, "per-page": CANDIDATES_PER_QUERY}, ) results = (data or {}).get("results", []) qualifying = [] for work in results: cand_title = work.get("title") or "" title_sim = fuzz.token_sort_ratio(title.lower(), cand_title.lower()) if title_sim < TITLE_THRESHOLD: continue if local_author: cand_names = _candidate_author_names(work) author_sim = max( (fuzz.ratio(local_author.lower(), n.lower()) for n in cand_names), default=0, ) if author_sim < AUTHOR_THRESHOLD: continue qualifying.append((work, title_sim)) if len(qualifying) == 1: work, title_sim = qualifying[0] return work, title_sim, len(results), 1 return None, 0, len(results), len(qualifying) def apply_match(item, work): doi = (work.get("doi") or "").replace("https://doi.org/", "").strip() if not doi: return False item.doi = doi item.openalex_id = work.get("id", "").replace("https://openalex.org/", "") or None item.cited_by_count = work.get("cited_by_count", 0) or 0 cby = work.get("counts_by_year") or [] item.counts_by_year = json.dumps(cby) if cby else None return True def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--apply", action="store_true", help="Write changes (default: dry run)") parser.add_argument("--limit", type=int, default=0, help="Max items to process (0 = all)") args = parser.parse_args() session = SessionLocal() try: q = ( session.query(Item) .filter(Item.doi.is_(None), Item.special_collection_score > 0) ) items = q.all() if args.limit: items = items[: args.limit] print("=" * 64) print(f"SC items missing DOI: {len(items)}") print("=" * 64) if not args.apply: print("[DRY RUN] No API calls or writes. Re-run with --apply.") return 0 matched = ambiguous = not_found = 0 for i, item in enumerate(items, 1): title = item.title or "" if not title.strip(): continue local_author = _first_author_name(item) work, title_sim, n_seen, n_qualifying = find_match(title, local_author) if work and apply_match(item, work): matched += 1 print( f" [{i}/{len(items)}] MATCH ({title_sim}%): " f"{title[:70]!r} -> {item.doi}" ) elif n_seen == 0: not_found += 1 else: ambiguous += 1 print( f" [{i}/{len(items)}] SKIP ({n_qualifying}/{n_seen} qualifying): " f"{title[:70]!r}" ) if i % 10 == 0: session.commit() time.sleep(0.5) # OpenAlex polite pool session.commit() analytics_cache.invalidate_all() print("\n" + "=" * 64) print( f"DONE. matched={matched} ambiguous/skipped={ambiguous} " f"not_found={not_found}" ) print("=" * 64) return 0 finally: session.close() if __name__ == "__main__": sys.exit(main())