Update fetch_paper_data.py

fetch_paper_data.py

@@ -1,71 +1,72 @@
-import re
-import requests
-from bs4 import BeautifulSoup
-from xml.etree import ElementTree as ET
-import json
-
-HEADERS = {
-    'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/131.0.6778.135 Safari/537.36'
-}
-
-def fetch_pmc_doi(pmc_id):
-    url = f"https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?email=raannakasturi@gmail.com&ids={pmc_id}&format=json"
-    response = requests.get(url, headers=HEADERS).json()
-    if response['status'] == 'ok':
-        doi = response['records'][0]['doi']
-        return f"https://doi.org/{doi}"
-
-def fetch_pmc_pdf(pmc_id):
-    url = f"https://www.ncbi.nlm.nih.gov/pmc/utils/oa/oa.fcgi?id={pmc_id}&format=pdf"
-    response = requests.get(url, headers=HEADERS).content
-    records = ET.fromstring(response).find('records').findall('record')
-    for record in records:
-        if record.attrib['id'] == pmc_id:
-            pdf_url = record.find('link').attrib['href']
-            return pdf_url.replace('ftp://', 'https://')
-        else:
-            return None
-
-def fetch_arxiv_doi(arxiv_id):
-    page_url = f"https://arxiv.org/abs/{arxiv_id}"
-    page_content = requests.get(page_url, headers=HEADERS).content
-    page_data = BeautifulSoup(page_content, 'html.parser')
-    doi = page_data.find('td', {'class': "tablecell arxivdoi"}).find('a', {'id': 'arxiv-doi-link'}).text
-    return doi
-
-def fetch_citation(doi):
-    citation_content = requests.get(doi, headers={ 'User-Agent':HEADERS['User-Agent'], 'Accept': 'text/x-bibliography; style=apa'}).content
-    return citation_content.decode('utf-8')
-
-def fetch_title(doi):
-    title_content = requests.get(doi, headers={ 'User-Agent':HEADERS['User-Agent'], 'Accept': 'text/x-bibliography; style=bibtex'}).content
-    bibtex_entry = title_content.decode('utf-8').strip()
-    title = re.search(r'title\s*=\s*{(.*?)}', bibtex_entry)
-    if title:
-        return title.group(1)
-    return None
-
-def fetch_paper_data(id):
-    data = {}
-    try:
-        if id.startswith('PMC'):
-            doi = fetch_pmc_doi(id)
-            pdf_url = fetch_pmc_pdf(id)
-        else:
-            doi = fetch_arxiv_doi(id)
-            pdf_url = f"https://arxiv.org/pdf/{id}"
-        citation = fetch_citation(doi).replace('\n', ' ').strip()
-        title = fetch_title(doi).replace('\n', ' ').strip()
-        data['status'] = 'success'
-        data['doi'] = doi
-        data['title'] = title
-        data['pdf_url'] = pdf_url
-        data['citation'] = citation
-    except Exception as e:
-        data['status'] = 'error'
-        print(str(e))
-    return json.dumps(data, indent=4, ensure_ascii=False)
-
-if __name__ == '__main__':
-    citation = fetch_paper_data('PMC8391798')
+import re
+import requests
+from bs4 import BeautifulSoup
+from xml.etree import ElementTree as ET
+import json
+
+HEADERS = {
+    'User-Agent': 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/131.0.6778.135 Safari/537.36'
+}
+
+def fetch_pmc_doi(pmc_id):
+    url = f"https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?email=raannakasturi@gmail.com&ids={pmc_id}&format=json"
+    response = requests.get(url, headers=HEADERS).json()
+    if response['status'] == 'ok':
+        doi = response['records'][0]['doi']
+        return f"https://doi.org/{doi}"
+
+def fetch_pmc_pdf(pmc_id):
+    url = f"https://www.ncbi.nlm.nih.gov/pmc/utils/oa/oa.fcgi?id={pmc_id}&format=pdf"
+    response = requests.get(url, headers=HEADERS).content
+    records = ET.fromstring(response).find('records').findall('record')
+    for record in records:
+        if record.attrib['id'] == pmc_id:
+            pdf_url = record.find('link').attrib['href']
+            return pdf_url.replace('ftp://', 'https://')
+        else:
+            return None
+
+def fetch_arxiv_doi(arxiv_id):
+    page_url = f"https://arxiv.org/abs/{arxiv_id}"
+    page_content = requests.get(page_url, headers=HEADERS).content
+    page_data = BeautifulSoup(page_content, 'html.parser')
+    doi = page_data.find('td', {'class': "tablecell arxivdoi"}).find('a', {'id': 'arxiv-doi-link'}).text
+    return doi
+
+def fetch_citation(doi):
+    citation_content = requests.get(doi, headers={ 'User-Agent':HEADERS['User-Agent'], 'Accept': 'text/x-bibliography; style=apa'}).content
+    return citation_content.decode('utf-8')
+
+def fetch_title(doi):
+    title_content = requests.get(doi, headers={ 'User-Agent':HEADERS['User-Agent'], 'Accept': 'text/x-bibliography; style=bibtex'}).content
+    bibtex_entry = title_content.decode('utf-8').strip()
+    title = re.search(r'title\s*=\s*{(.*?)}', bibtex_entry)
+    if title:
+        return title.group(1)
+    return None
+
+def fetch_paper_data(id):
+    data = {}
+    try:
+        if id.startswith('PMC'):
+            doi = fetch_pmc_doi(id)
+            pdf_url = fetch_pmc_pdf(id)
+        else:
+            doi = fetch_arxiv_doi(id)
+            pdf_url = f"https://arxiv.org/pdf/{id}"
+        citation = fetch_citation(doi).replace('\n', ' ').strip()
+        title = fetch_title(doi).replace('\n', ' ').strip()
+        data['status'] = 'success'
+        data['data'] = {}
+        data['data']['doi'] = doi
+        data['data']['title'] = title
+        data['data']['pdf_url'] = pdf_url
+        data['data']['citation'] = citation
+    except Exception as e:
+        data['status'] = 'error'
+        print(str(e))
+    return json.dumps(data, indent=4, ensure_ascii=False)
+
+if __name__ == '__main__':
+    citation = fetch_paper_data('PMC8391798')
     print(citation)