Update tools.py

tools.py

@@ -8,20 +8,16 @@ from sklearn.decomposition import PCA
 from sklearn.metrics.pairwise import cosine_similarity
 from sklearn.cluster import AgglomerativeClustering
 
-# --- Persistent Storage Configuration ---
-# Hugging Face Spaces typically mounts persistent data to /data
-PERSISTENT_BASE = "/data" if os.path.exists("/data") else os.path.dirname(os.path.abspath(__file__))
+# Use /data for persistence if running on HF Spaces with a mounted bucket, otherwise use local directory
+ROOT_DIR = "/data" if os.path.exists("/data") else os.path.dirname(os.path.abspath(__file__))
 
-CSV_PATH  = os.path.join(PERSISTENT_BASE, "papers.csv")
-OUT_DIR   = os.path.join(PERSISTENT_BASE, "outputs")
-PDF_DIR   = os.path.join(PERSISTENT_BASE, "pdfs")
-SCRATCH_DIR = os.path.join(PERSISTENT_BASE, "scratch")
+CSV_PATH  = os.path.join(ROOT_DIR, "papers.csv")
+OUT_DIR   = os.path.join(ROOT_DIR, "outputs")
+PDF_DIR   = os.path.join(ROOT_DIR, "pdfs")
+SCRATCH_DIR = os.path.join(ROOT_DIR, "scratch")
+for _d in [OUT_DIR, PDF_DIR, SCRATCH_DIR]: os.makedirs(_d, exist_ok=True)
 
-# Ensure all persistent directories exist
-for _d in [OUT_DIR, PDF_DIR, SCRATCH_DIR]: 
-    os.makedirs(_d, exist_ok=True)
-
-HEADERS = ["Sr No", "Title", "DOI", "Web Link", "Authors", "Date of Publication", "Journal", "Abstract", "No of Citations", "Keywords"]
+HEADERS = ["Sr No", "Title", "DOI", "Web Link", "Authors", "Date of Publication", "Journal", "Abstract", "No of Citations"]
 
 PAJAIS_TAXONOMY = {
     "AI & Machine Learning":        ["deep learning","neural networks","NLP","computer vision","reinforcement learning","transformers"],
@@ -46,7 +42,7 @@ def _embed():
 
 @tool
 def search_academic_source(query: str, source: str) -> str:
-    """Search academic sources. Valid sources: 'google_scholar', 'arxiv', 'tavily'."""
+    """Search academic sources. Valid sources: 'google_scholar', 'arxiv', 'pubmed', 'hf_papers', 'tavily', 'apify', 'semantic_scholar', 'openalex', 'scopus', 'web_of_science'."""
     try:
         src = source.lower()
         if "google" in src or "scholar" in src:
@@ -55,25 +51,52 @@ def search_academic_source(query: str, source: str) -> str:
         elif "arxiv" in src:
             from langchain_community.utilities import ArxivAPIWrapper
             return ArxivAPIWrapper(top_k_results=5).run(query)[:5000]
+        elif "pubmed" in src:
+            from langchain_community.utilities.pubmed import PubMedAPIWrapper
+            return PubMedAPIWrapper(top_k_results=5).run(query)[:5000]
+        elif "hf_papers" in src or "huggingface" in src:
+            res = requests.get(f"https://huggingface.co/api/papers/search?q={urllib.parse.quote(query)}").json()
+            return str(res[:5])[:5000] if isinstance(res, list) else "HF Search Error"
+        elif "apify" in src:
+            from langchain_community.utilities import ApifyWrapper
+            apify = ApifyWrapper()
+            return apify.run("apify/arxiv-scraper", run_input={"searchSubj": "all", "searchQuery": query, "maxResults": 5})[:5000]
+        elif "semantic_scholar" in src:
+            from langchain_community.utilities.semanticscholar import SemanticScholarAPIWrapper
+            return SemanticScholarAPIWrapper(top_k_results=5).run(query)[:5000]
+        elif "openalex" in src:
+            api_key = os.getenv("OPENALEX_API_KEY")
+            url = f"https://api.openalex.org/works?search={urllib.parse.quote(query)}&per-page=5"
+            headers = {"User-Agent": "AcademicAgent/1.0 (mailto:agent@research-platform.org)"}
+            if api_key: headers["api_key"] = api_key # Higher rate limits if key provided
+            res = requests.get(url, headers=headers).json()
+            return str(res.get("results", []))[:5000]
+        elif "scopus" in src:
+            api_key = os.getenv("SCOPUS_API_KEY")
+            if not api_key: return "Error: SCOPUS_API_KEY missing. Request the user to provide it or switch to 'semantic_scholar' or 'openalex'."
+            res = requests.get(f"https://api.elsevier.com/content/search/scopus?query={urllib.parse.quote(query)}&count=5", headers={"X-ELS-APIKey": api_key, "Accept": "application/json"}).json()
+            return str(res)[:5000]
+        elif "web_of_science" in src or "wos" in src:
+            api_key = os.getenv("WOS_API_KEY")
+            if not api_key: return "Error: WOS_API_KEY missing. Request the user to provide it or switch to 'semantic_scholar' or 'openalex'."
+            res = requests.get(f"https://wos-api.clarivate.com/api/wos?databaseId=WOS&usrQuery={urllib.parse.quote(query)}&count=5&firstRecord=1", headers={"X-ApiKey": api_key}).json()
+            return str(res)[:5000]
         else:
             from langchain_community.tools.tavily_search import TavilySearchResults
-            # Tavily is an LLM-friendly in-built web searcher
-            return str(TavilySearchResults(max_results=5).invoke(f"{query} academic paper abstract"))[:5000]
+            return str(TavilySearchResults(max_results=5).invoke(f"{query} academic peer reviewed paper"))[:5000]
     except Exception as e: return f"Error searching {source}: {str(e)}"
 
-@tool
-def download_pdf(url: str, filename: str) -> str:
-    """Download a PDF from the given URL and save it to the pdfs/ folder. Ensure filename ends with .pdf."""
+def _download_pdf(url: str, filename: str):
     try:
         r = requests.get(url, stream=True, timeout=12)
         if r.status_code == 200:
             if not filename.lower().endswith(".pdf"): filename += ".pdf"
-            path = os.path.join(PDF_DIR, filename)
+            path = os.path.join(PDF_DIR, "".join(x for x in filename if x.isalnum() or x in "._- "))
             with open(path, 'wb') as f:
                 for chunk in r.iter_content(chunk_size=8192): f.write(chunk)
-            return f"Success: Downloaded to {path}"
-        return f"Failed: HTTP {r.status_code}"
-    except Exception as e: return f"Download error: {e}"
+            return True
+    except: pass
+    return False
 
 @tool
 def save_papers(papers_json: str) -> str:
@@ -92,8 +115,15 @@ def save_papers(papers_json: str) -> str:
             impact_words = ["societ", "ethic", "bias", "fairness", "privacy", "impact", "policy", "govern", "regulation", "human"]
             return any(w in txt for w in ai_words) and any(w in txt for w in impact_words)
             
-        new_rows = [p for p in incoming if p.get("Title","").strip().lower() not in seen and is_relevant(p)]
-        discarded_count = len([p for p in incoming if p.get("Title","").strip().lower() not in seen and not is_relevant(p)])
+        # Strict Academic Filter: Discard Non-Papers
+        def is_research_paper(p):
+            txt = (str(p.get("Title", "")) + " " + str(p.get("Journal", "")) + " " + str(p.get("Abstract", ""))).lower()
+            banned_words = ["book", "chapter", "news", "editorial", "encyclopedia", "newspaper", "magazine"]
+            return not any(w in txt.split() for w in banned_words)  # split() guarantees strict word matching
+            
+        valid_items = [p for p in incoming if p.get("Title","").strip().lower() not in seen and is_relevant(p) and is_research_paper(p)]
+        discarded_count = len([p for p in incoming if p.get("Title","").strip().lower() not in seen and not (is_relevant(p) and is_research_paper(p))])
+        new_rows = valid_items
         
         # Enforce MAX_TOTAL_PAPERS = 120 (Prioritize existing CSV papers)
         total_current = len(existing)
@@ -107,8 +137,18 @@ def save_papers(papers_json: str) -> str:
         
         if final_new_rows:
             pd.concat([existing, pd.DataFrame(final_new_rows)]).to_csv(CSV_PATH, index=False)
+            
+            # Auto-download PDFs for new rows
+            download_count = 0
+            for p in final_new_rows:
+                link = p.get("Web Link", "")
+                if link and str(link).lower().endswith(".pdf"):
+                    if _download_pdf(link, p.get("Title", "paper")[:50]):
+                        download_count += 1
+
             enrich_doi.invoke({}) # Auto-fill DOIs for new papers
             msg = f"Saved {len(final_new_rows)} papers. Total: {len(existing)+len(final_new_rows)}."
+            if download_count > 0: msg += f" (Archived {download_count} PDFs)."
             if discarded_count > 0: msg += f" (Discarded {discarded_count} papers for not aligning with AI & Societal Impact)."
             if len(new_rows) > allowed_new: msg += f" (Truncated {len(new_rows) - allowed_new} exceeding cap)."
             return msg