Update fetch_citation to use new API endpoint and improved headers for citation retrieval

fetch_paper_data.py

@@ -32,8 +32,27 @@ def fetch_arxiv_doi(arxiv_id):
     return doi
 
 def fetch_citation(doi):
-    citation_content = requests.get(doi, headers={ 'User-Agent':HEADERS['User-Agent'], 'Accept': 'text/x-bibliography; style=apa'}).content
-    return citation_content.decode('utf-8')
+    url = f"https://citation.crosscite.org/format?doi={doi}&style=apa&lang=en-US"
+    headers = {
+        "accept": "text/plain, */*; q=0.01",
+        "accept-language": "en-US,en-GB;q=0.9,en;q=0.8",
+        "priority": "u=1, i",
+        "sec-ch-ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
+        "sec-ch-ua-mobile": "?1",
+        "sec-ch-ua-platform": '"Android"',
+        "sec-fetch-dest": "empty",
+        "sec-fetch-mode": "cors",
+        "sec-fetch-site": "same-origin",
+        "x-requested-with": "XMLHttpRequest",
+        "Referer": "https://citation.crosscite.org/",
+        "Referrer-Policy": "strict-origin-when-cross-origin"
+    }
+    
+    response = requests.get(url, headers=headers)
+    if response.status_code == 200:
+        return response.text
+    else:
+        response.raise_for_status()
 
 def fetch_title(doi):
     title_content = requests.get(doi, headers={ 'User-Agent':HEADERS['User-Agent'], 'Accept': 'text/x-bibliography; style=bibtex'}).content
@@ -68,4 +87,4 @@ def fetch_paper_data(id):
 
 if __name__ == '__main__':
     data = fetch_paper_data('PMC317040')
-    print(data)
+    print(data)