Update smart_fallback.py

smart_fallback.py

@@ -4,6 +4,8 @@ import mtdna_classifier
 from NER.html import extractHTML
 import data_preprocess
 import pipeline
+import aiohttp
+import asyncio
 # Setup
 def fetch_ncbi(accession_number):
   try:
@@ -242,6 +244,63 @@ def smart_google_queries(metadata: dict):
 #     if better_filter:
 #       filtered = better_filter      
 #     return filtered
+async def process_link(session, link, saveLinkFolder, keywords, accession):
+    output = []
+    title_snippet = link.lower()
+
+    # use async extractor for web, fallback to sync for local files
+    if link.startswith("http"):
+        article_text = await data_preprocess.async_extract_text(link, saveLinkFolder)
+    else:
+        article_text = data_preprocess.extract_text(link, saveLinkFolder)
+
+    for keyword in keywords:
+        if article_text and keyword.lower() in article_text.lower():
+            output.append([link, keyword.lower(), article_text])
+            return output
+        if keyword.lower() in title_snippet:
+            output.append([link, keyword.lower()])
+            return output
+    return output
+
+async def async_filter_links_by_metadata(search_results, saveLinkFolder, accession=None):
+    TRUSTED_DOMAINS = [
+        "ncbi.nlm.nih.gov", "pubmed.ncbi.nlm.nih.gov", "pmc.ncbi.nlm.nih.gov",
+        "biorxiv.org", "researchgate.net", "nature.com", "sciencedirect.com"
+    ]
+
+    keywords = ["mtDNA", "mitochondrial", "accession", "isolate", "Homo sapiens", "sequence"]
+    if accession:
+        keywords = [accession] + keywords
+
+    filtered, better_filter = {}, {}
+
+    async with aiohttp.ClientSession() as session:
+        tasks = []
+        for link in search_results:
+            if link:
+                tasks.append(process_link(session, link, saveLinkFolder, keywords, accession))
+
+        results = await asyncio.gather(*tasks)
+
+    # merge results
+    for output_link in results:
+        for out_link in output_link:
+            if isinstance(out_link, list) and len(out_link) > 1:
+                kw = out_link[1]
+                if accession and kw == accession.lower():
+                    if len(out_link) == 2:
+                        better_filter[out_link[0]] = ""
+                    elif len(out_link) == 3:
+                        better_filter[out_link[0]] = out_link[2]
+                if len(out_link) == 2:
+                    better_filter[out_link[0]] = ""
+                elif len(out_link) == 3:
+                    better_filter[out_link[0]] = out_link[2]
+            else:
+                filtered[out_link] = ""
+
+    return better_filter or filtered
 
 def filter_links_by_metadata(search_results, saveLinkFolder, accession=None):
     TRUSTED_DOMAINS = [