Update smart_fallback.py

smart_fallback.py

@@ -147,7 +147,7 @@ def smart_google_queries(metadata: dict):
           
     return queries
 
-def filter_links_by_metadata(search_results, saveLinkFolder, accession=None):
+def filter_links_by_metadata(search_results, saveLinkFolder, accession=None, stop_flag=None):
     TRUSTED_DOMAINS = [
     "ncbi.nlm.nih.gov",
     "pubmed.ncbi.nlm.nih.gov",
@@ -157,6 +157,9 @@ def filter_links_by_metadata(search_results, saveLinkFolder, accession=None):
     "nature.com",
     "sciencedirect.com"
     ]
+    if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop detected {accession}, aborting early...")
+        return []
     def is_trusted_link(link):
       for domain in TRUSTED_DOMAINS:
         if domain in link:
@@ -170,6 +173,9 @@ def filter_links_by_metadata(search_results, saveLinkFolder, accession=None):
       title_snippet = link.lower()
       print("save link folder inside this filter function: ", saveLinkFolder)  
       success_process, output_process = pipeline.run_with_timeout(data_preprocess.extract_text,args=(link,saveLinkFolder),timeout=60)
+      if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop detected {accession}, aborting early...")
+        return []
       if success_process:
           article_text = output_process
           print("yes succeed for getting article text")
@@ -179,10 +185,16 @@ def filter_links_by_metadata(search_results, saveLinkFolder, accession=None):
       #article_text = data_preprocess.extract_text(link,saveLinkFolder)
       print("article text")
       #print(article_text)  
+      if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop detected {accession}, aborting early...")
+        return []  
       try:
         ext = link.split(".")[-1].lower()
         if ext not in ["pdf", "docx", "xlsx"]:
             html = extractHTML.HTML("", link)
+            if stop_flag is not None and stop_flag.value:
+                print(f"🛑 Stop detected {accession}, aborting early...")
+                return []
             jsonSM = html.getSupMaterial()
             if jsonSM:
                 output += sum((jsonSM[key] for key in jsonSM), [])
@@ -210,6 +222,9 @@ def filter_links_by_metadata(search_results, saveLinkFolder, accession=None):
           #     filtered.append(link)
           # else:
           print(link)
+          if stop_flag is not None and stop_flag.value:
+            print(f"🛑 Stop detected {accession}, aborting early...")
+            return []
           if link:    
             output_link = is_relevant_title_snippet(link,saveLinkFolder, accession)
             print("output link: ")