Update dvnc_ai_v2_hf/self_learning_graph.py

dvnc_ai_v2_hf/self_learning_graph.py

@@ -1,14 +1,10 @@
 import html
 import json
-import math
-import os
 import re
-import time
 import urllib.parse
 import xml.etree.ElementTree as ET
-from collections import Counter
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
 
 import gradio as gr
 import requests
@@ -18,24 +14,16 @@ try:
 except Exception:
     fitz = None
 
-try:
-    from bs4 import BeautifulSoup
-except Exception:
-    BeautifulSoup = None
 
+SEARCH_MODES = ["topic", "title", "doi", "link", "paper_name", "autonomous_web"]
+SOURCE_OPTIONS = ["arxiv", "crossref", "openalex", "semantic_scholar", "europe_pmc"]
+DEFAULT_SOURCES = ["arxiv", "crossref", "openalex", "semantic_scholar", "europe_pmc"]
+PDF_PARSERS = ["pymupdf"]
 
-def build_learning_graph_html(nodes, edges, title="Self-Learning Knowledge Graph"):
-    return f"""
-    <div class="panel">
-      <h3>{title}</h3>
-      <p>Nodes: {len(nodes or [])}</p>
-      <p>Edges: {len(edges or [])}</p>
-    </div>
-    """
-
-# ---------------------------------------------------------------------------
-# Constants
-# ---------------------------------------------------------------------------
+REQUEST_TIMEOUT = 25
+GRAPH_MAX_RESULTS = 10
+GRAPH_MAX_CONCEPTS = 10
+GRAPH_MAX_CLAIMS = 6
 
 JOURNALS = [
     {"name": "Nature", "url": "https://www.nature.com/search", "desc": "Flagship multidisciplinary research journal."},
@@ -45,41 +33,24 @@ JOURNALS = [
     {"name": "IEEE Xplore", "url": "https://ieeexplore.ieee.org/search/searchresult.jsp", "desc": "Engineering, AI, signal processing, and systems."},
 ]
 
-SEARCH_MODES = ["topic", "title", "doi", "link", "paper_name", "autonomous_web"]
-SOURCE_OPTIONS = ["arxiv", "crossref", "openalex", "semantic_scholar", "europe_pmc"]
-DEFAULT_SOURCES = ["arxiv", "crossref", "openalex", "semantic_scholar", "europe_pmc"]
-
-SEMANTIC_SCHOLAR_API_KEY = os.getenv("SEMANTIC_SCHOLAR_API_KEY", "").strip()
-GROBID_URL = os.getenv("GROBID_URL", "").strip()
-REQUEST_TIMEOUT = int(os.getenv("REQUEST_TIMEOUT", "25"))
-GRAPH_MAX_CONCEPTS = int(os.getenv("GRAPH_MAX_CONCEPTS", "10"))
-GRAPH_MAX_CLAIMS = int(os.getenv("GRAPH_MAX_CLAIMS", "6"))
-GRAPH_MAX_RESULTS = int(os.getenv("GRAPH_MAX_RESULTS", "10"))
-
 STOPWORDS = {
     "a", "an", "and", "are", "as", "at", "be", "been", "being", "by", "can", "could",
     "did", "do", "does", "for", "from", "had", "has", "have", "if", "in", "into", "is",
     "it", "its", "may", "might", "of", "on", "or", "our", "such", "that", "the", "their",
     "there", "these", "this", "those", "to", "using", "use", "used", "via", "was", "were",
     "will", "with", "within", "without", "we", "they", "you", "your", "study", "paper",
-    "research", "results", "method", "methods", "analysis", "approach", "toward", "towards",
-    "based", "new", "novel", "effect", "effects", "model", "models", "system", "systems",
-    "show", "shows",
+    "research", "results", "result", "method", "methods", "analysis", "approach", "based",
+    "new", "novel", "effect", "effects", "model", "models", "system", "systems", "show",
+    "shows", "shown", "however", "therefore", "also", "between", "across", "among", "et", "al",
 }
 
 GRAPH_MEMORY = {
     "papers": {},
     "nodes": {},
     "edges": [],
-    "concept_counts": Counter(),
-    "claim_counts": Counter(),
     "queries": [],
-    "events": [],
 }
 
-# ---------------------------------------------------------------------------
-# Text utilities
-# ---------------------------------------------------------------------------
 
 def safe_text(x, default=""):
     return html.escape(str(x if x is not None else default))
@@ -89,18 +60,23 @@ def norm_text(x: Optional[str]) -> str:
     return re.sub(r"\s+", " ", (x or "")).strip()
 
 
-def slugify(text: str) -> str:
-    return re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-")
-
-
 def ensure_list(x):
     return x if isinstance(x, list) else []
 
 
+def truncate_text(text: str, limit: int) -> str:
+    text = norm_text(text)
+    return text if len(text) <= limit else text[: limit - 1].rstrip() + "…"
+
+
+def slugify(text: str) -> str:
+    return re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-")
+
+
 def normalize_doi(text: str) -> str:
     text = (text or "").strip()
-    text = text.replace("https://doi.org/", "").replace("http://doi.org/", "")
-    return text.strip()
+    text = re.sub(r"^https?://(dx\.)?doi\.org/", "", text, flags=re.I)
+    return text.strip().rstrip("/")
 
 
 def detect_query_type(query: str) -> str:
@@ -114,171 +90,214 @@ def detect_query_type(query: str) -> str:
 
 
 def tokenize(text: str) -> List[str]:
-    return [t for t in re.findall(r"[a-zA-Z][a-zA-Z0-9\-]{2,}", (text or "").lower()) if t not in STOPWORDS]
+    return [
+        t for t in re.findall(r"[A-Za-z][A-Za-z0-9\-/+]{2,}", (text or ""))
+        if t.lower() not in STOPWORDS
+    ]
 
 
-def unique_keep_order(items: List[str]) -> List[str]:
-    seen = set()
-    out = []
-    for item in items:
-        key = norm_text(item).lower()
-        if key and key not in seen:
-            seen.add(key)
-            out.append(norm_text(item))
-    return out
+def clean_pdf_text(text: str) -> str:
+    text = text or ""
+    replacements = {
+        "\u00ad": "",
+        "\ufb01": "fi",
+        "\ufb02": "fl",
+        "\u2010": "-",
+        "\u2011": "-",
+        "\u2012": "-",
+        "\u2013": "-",
+        "\u2014": "-",
+        "\u2212": "-",
+        "\u00a0": " ",
+    }
+    for old, new in replacements.items():
+        text = text.replace(old, new)
 
+    text = re.sub(r"(\w)-\s*\n\s*(\w)", r"\1\2", text)
+    text = re.sub(r"([a-z])\n([a-z])", r"\1 \2", text)
+    text = re.sub(r"\bnjectable\b", "injectable", text, flags=re.I)
+    text = re.sub(r"\bfhydrogel\b", "hydrogel", text, flags=re.I)
+    text = re.sub(r"\b[a-z](hydrogel|conductive|responsive|scaffold|biomaterial|regeneration)\b", r"\1", text, flags=re.I)
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
 
-def text_overlap_score(a: str, b: str) -> float:
-    sa = set(tokenize(a))
-    sb = set(tokenize(b))
-    if not sa or not sb:
-        return 0.0
-    return len(sa & sb) / max(1, len(sa | sb))
 
+def normalize_phrase(phrase: str) -> str:
+    phrase = norm_text(phrase)
+    mapping = {"ph": "pH", "ai": "AI", "ml": "ML", "3d": "3D"}
+    out = []
+    for part in phrase.split():
+        out.append(mapping.get(part.lower(), part))
+    return " ".join(out)
 
-def compute_recency_bonus(year: str) -> float:
-    try:
-        y = int(str(year)[:4])
-    except Exception:
-        return 0.0
-    current = time.gmtime().tm_year
-    age = max(current - y, 0)
-    return max(0.0, 0.12 - age * 0.015)
 
+def looks_bad_phrase(phrase: str) -> bool:
+    phrase = norm_text(phrase)
+    if not phrase or len(phrase) < 4 or len(phrase) > 80:
+        return True
+    parts = phrase.split()
+    if len(parts) > 5:
+        return True
+    for p in parts:
+        stripped = p.strip("-.,;:()[]{}")
+        if len(stripped) == 1:
+            return True
+        if re.match(r"^[bcdfghjklmnpqrstvwxyz]{5,}$", stripped.lower()):
+            return True
+    return False
+
+
+def extract_concepts_from_text(text: str, max_terms: int = GRAPH_MAX_CONCEPTS) -> List[str]:
+    text = clean_pdf_text(text)
+    words = re.findall(r"[A-Za-z][A-Za-z0-9\-/+]{2,}", text)
+    candidates = []
 
-def extract_candidate_phrases(text: str, max_terms: int = 20) -> List[str]:
-    text = norm_text(text)
-    if not text:
-        return []
-    tokens = re.findall(r"[A-Za-z][A-Za-z0-9\-]{2,}", text)
-    phrases = []
     for n in (3, 2, 1):
-        for i in range(len(tokens) - n + 1):
-            phrase = " ".join(tokens[i:i + n]).strip().lower()
-            if len(phrase) < 4:
+        for i in range(len(words) - n + 1):
+            phrase = " ".join(words[i:i + n])
+            low = phrase.lower()
+            if any(tok in STOPWORDS for tok in low.split()):
                 continue
-            parts = phrase.split()
-            if any(p in STOPWORDS for p in parts):
+            if looks_bad_phrase(low):
                 continue
-            if all(len(p) <= 2 for p in parts):
-                continue
-            phrases.append(phrase)
-    counts = Counter(phrases)
-    ranked = [p for p, _ in counts.most_common(max_terms * 3)]
-    filtered = []
-    for phrase in ranked:
-        if phrase in filtered:
+            candidates.append(low)
+
+    ranked = []
+    counts = {}
+    for phrase in candidates:
+        counts[phrase] = counts.get(phrase, 0) + 1
+
+    for phrase, count in sorted(counts.items(), key=lambda x: x[1], reverse=True):
+        score = count + 0.2 * len(phrase.split())
+        if any(k in phrase for k in ["hydrogel", "conductive", "responsive", "injectable", "graph", "neural", "cardiac", "biomaterial"]):
+            score += 1.0
+        ranked.append((score, phrase))
+
+    output = []
+    seen = set()
+    for _, phrase in sorted(ranked, key=lambda x: x[0], reverse=True):
+        clean = normalize_phrase(phrase)
+        low = clean.lower()
+        if low in seen:
             continue
-        if any(phrase != other and phrase in other for other in filtered):
+        if any((low in s or s in low) and abs(len(low) - len(s)) <= 2 for s in seen):
             continue
-        filtered.append(phrase)
-        if len(filtered) >= max_terms:
+        seen.add(low)
+        output.append(clean)
+        if len(output) >= max_terms:
             break
-    return filtered
-
 
-def extract_concepts_from_text(text: str, max_terms: int = GRAPH_MAX_CONCEPTS) -> List[str]:
-    return extract_candidate_phrases(text, max_terms=max_terms)
+    return output
 
 
 def extract_claim_like_sentences(text: str, max_items: int = GRAPH_MAX_CLAIMS) -> List[str]:
-    text = norm_text(text)
-    if not text:
-        return []
-    parts = re.split(r"(?<=[.!?])\s+", text)
+    text = clean_pdf_text(text)
+    sentences = [norm_text(s) for s in re.split(r"(?<=[\.\!\?])\s+", text) if norm_text(s)]
     scored = []
-    for sentence in parts:
-        s = norm_text(sentence)
-        if len(s) < 40 or len(s) > 280:
+
+    for sentence in sentences:
+        if len(sentence) < 40 or len(sentence) > 320:
             continue
-        lower = s.lower()
+        lower = sentence.lower()
         score = 0.0
-        if any(k in lower for k in ["improves", "reduces", "increases", "suggests", "demonstrates",
-                                     "shows", "reveals", "predicts", "achieves", "outperforms"]):
+        if any(k in lower for k in ["improves", "reduces", "increases", "demonstrates", "shows", "reveals", "predicts", "achieves", "outperforms", "enables"]):
             score += 2.0
-        if any(k in lower for k in ["significant", "associated", "correlated", "effective",
-                                     "robust", "accurate", "validated"]):
+        if any(k in lower for k in ["significant", "associated", "correlated", "effective", "robust", "accurate", "validated"]):
             score += 1.0
-        score += min(len(tokenize(s)) / 15.0, 2.0)
-        scored.append((score, s))
-    return [s for _, s in sorted(scored, key=lambda x: x[0], reverse=True)[:max_items]]
-
-
-def parse_openalex_abstract(inverted_index) -> str:
-    if not inverted_index or not isinstance(inverted_index, dict):
-        return ""
-    pos_to_word = {}
-    for word, positions in inverted_index.items():
-        for pos in positions:
-            pos_to_word[pos] = word
-    if not pos_to_word:
-        return ""
-    return " ".join(pos_to_word[i] for i in sorted(pos_to_word))
-
-# ---------------------------------------------------------------------------
-# Enrichment
-# ---------------------------------------------------------------------------
-
-def enrich_paper_semantics(query: str, paper: Dict) -> Dict:
+        score += min(len(tokenize(sentence)) / 18.0, 2.0)
+        scored.append((score, sentence))
+
+    scored.sort(key=lambda x: x[0], reverse=True)
+    out = []
+    seen = set()
+    for _, sentence in scored:
+        key = sentence.lower()
+        if key not in seen:
+            seen.add(key)
+            out.append(sentence)
+        if len(out) >= max_items:
+            break
+    return out
+
+
+def extract_title_from_text(text: str, fallback: str = "Uploaded PDF") -> str:
+    text = clean_pdf_text(text)
+    lines = [norm_text(x) for x in text.splitlines() if norm_text(x)]
+    for line in lines[:18]:
+        if 8 <= len(line) <= 220 and not re.search(r"doi|http|www\.|@", line, flags=re.I):
+            return truncate_text(line.strip(" -:;"), 240)
+    return fallback
+
+
+def extract_references_from_text(text: str) -> List[Dict[str, str]]:
+    refs = []
+    seen = set()
+    doi_re = re.compile(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", re.I)
+
+    for line in [norm_text(x) for x in clean_pdf_text(text).splitlines() if norm_text(x)][-250:]:
+        doi_match = doi_re.search(line)
+        doi = normalize_doi(doi_match.group(0)) if doi_match else ""
+        title = line.replace(doi_match.group(0), "").strip(" .;,-") if doi_match else line
+        if len(title) < 12:
+            continue
+        key = (title.lower(), doi.lower())
+        if key in seen:
+            continue
+        seen.add(key)
+        refs.append({"title": truncate_text(title, 220), "doi": doi})
+        if len(refs) >= 30:
+            break
+    return refs
+
+
+def score_overlap(query: str, text: str) -> float:
+    qa = {x.lower() for x in tokenize(query)}
+    ta = {x.lower() for x in tokenize(text)}
+    if not qa or not ta:
+        return 0.0
+    return len(qa & ta) / max(1, len(qa | ta))
+
+
+def enrich_paper(query: str, paper: Dict[str, Any]) -> Dict[str, Any]:
     paper = dict(paper)
-    title = paper.get("title", "")
-    abstract = paper.get("abstract", "") or paper.get("summary", "")
-    venue = paper.get("venue", "")
-    base_text = " ".join([title, abstract, venue]).strip()
-    concepts = extract_concepts_from_text(base_text, max_terms=GRAPH_MAX_CONCEPTS)
-    claims = extract_claim_like_sentences(abstract, max_items=GRAPH_MAX_CLAIMS)
-    rel = text_overlap_score(query, f"{title} {abstract}")
-    recency = compute_recency_bonus(paper.get("year"))
-    doi_bonus = 0.02 if paper.get("doi") else 0.0
-    oa_bonus = 0.03 if paper.get("open_access") else 0.0
-    concept_bonus = min(len(concepts), 8) * 0.01
-    learned_score = float(paper.get("score", 0)) + rel * 0.5 + recency + doi_bonus + oa_bonus + concept_bonus
-    paper["concepts"] = concepts[:GRAPH_MAX_CONCEPTS]
-    paper["claims"] = claims[:GRAPH_MAX_CLAIMS]
-    paper["relevance"] = round(rel, 4)
-    paper["learned_score"] = round(learned_score, 4)
+    title = clean_pdf_text(paper.get("title", ""))
+    abstract = clean_pdf_text(paper.get("abstract", "") or paper.get("summary", ""))
+    venue = clean_pdf_text(paper.get("venue", ""))
+    concepts = extract_concepts_from_text(f"{title} {abstract} {venue}", GRAPH_MAX_CONCEPTS)
+    claims = extract_claim_like_sentences(abstract, GRAPH_MAX_CLAIMS)
+    overlap = score_overlap(query, f"{title} {abstract}")
+    score = float(paper.get("score", 0)) + overlap * 0.5 + min(len(concepts), 8) * 0.02
+
+    paper["title"] = title or "Untitled"
+    paper["abstract"] = abstract
+    paper["summary"] = truncate_text(abstract or paper.get("summary", ""), 420)
+    paper["venue"] = venue
+    paper["concepts"] = concepts
+    paper["claims"] = claims
+    paper["learned_score"] = round(score, 4)
     return paper
 
 
-def paper_identity_key(paper: Dict) -> str:
-    return (
-        normalize_doi(paper.get("doi") or "")
-        or (paper.get("external_ids") or {}).get("arxiv")
-        or norm_text(paper.get("title", "")).lower()
-        or str(paper.get("id"))
-    )
+def paper_identity_key(paper: Dict[str, Any]) -> str:
+    return normalize_doi(paper.get("doi", "")) or norm_text(paper.get("title", "")).lower()
 
-# ---------------------------------------------------------------------------
-# Journal links
-# ---------------------------------------------------------------------------
 
-def journal_query_links(query: str):
-    q = urllib.parse.quote_plus(query or "biomaterials cardiac repair")
-    rows = []
-    for journal in JOURNALS:
-        url = f"{journal['url']}?q={q}" if "?" not in journal["url"] else f"{journal['url']}&q={q}"
-        if "ieeexplore" in journal["url"]:
-            url = f"https://ieeexplore.ieee.org/search/searchresult.jsp?queryText={q}"
-        rows.append({"name": journal["name"], "desc": journal["desc"], "url": url})
-    return rows
-
-
-def build_journal_html(query):
-    rows = []
-    for journal in journal_query_links(query):
-        rows.append(
-            f"""<div style="border:1px solid #e2e8f0;border-radius:8px;padding:12px 16px;margin:6px 0;display:flex;align-items:center;justify-content:space-between;background:#f8fafc;">
-<div><strong>{safe_text(journal['name'])}</strong><br/><span style="color:#64748b;font-size:0.85em;">{safe_text(journal['desc'])}</span></div>
-<a href="{safe_text(journal['url'])}" target="_blank" style="background:#6366f1;color:#fff;padding:6px 14px;border-radius:6px;text-decoration:none;font-size:0.85em;">Open ↗</a>
-</div>"""
-        )
-    return '<div style="padding:8px 0;">' + "".join(rows) + "</div>"
+def dedupe_papers(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    seen = {}
+    for item in items:
+        key = paper_identity_key(item) or f"{item.get('source', 'src')}::{item.get('title', 'paper')}"
+        current = seen.get(key)
+        cur_score = float(current.get("learned_score", current.get("score", 0))) if current else -1
+        new_score = float(item.get("learned_score", item.get("score", 0)))
+        if current is None or new_score > cur_score:
+            seen[key] = item
+    out = list(seen.values())
+    out.sort(key=lambda x: float(x.get("learned_score", x.get("score", 0))), reverse=True)
+    return out
 
-# ---------------------------------------------------------------------------
-# Source search functions
-# ---------------------------------------------------------------------------
 
-def search_arxiv(query, max_results=8):
+def search_arxiv(query: str, max_results: int = 6) -> List[Dict[str, Any]]:
     encoded = urllib.parse.quote(query)
     url = (
         "http://export.arxiv.org/api/query?search_query=all:"
@@ -286,20 +305,26 @@ def search_arxiv(query, max_results=8):
     )
     response = requests.get(url, timeout=REQUEST_TIMEOUT)
     response.raise_for_status()
+
     root = ET.fromstring(response.text)
     ns = {"atom": "http://www.w3.org/2005/Atom"}
     papers = []
+
     for entry in root.findall("atom:entry", ns):
-        title = " ".join((entry.findtext("atom:title", default="", namespaces=ns) or "").split())
-        summary = " ".join((entry.findtext("atom:summary", default="", namespaces=ns) or "").split())
-        published = entry.findtext("atom:published", default="", namespaces=ns)
-        paper_id = entry.findtext("atom:id", default="", namespaces=ns)
-        authors = [a.findtext("atom:name", default="", namespaces=ns) for a in entry.findall("atom:author", ns)]
+        title = clean_pdf_text(entry.findtext("atom:title", default="", namespaces=ns) or "")
+        summary = clean_pdf_text(entry.findtext("atom:summary", default="", namespaces=ns) or "")
+        published = entry.findtext("atom:published", default="", namespaces=ns) or ""
+        paper_id = entry.findtext("atom:id", default="", namespaces=ns) or ""
+        authors = [
+            clean_pdf_text(a.findtext("atom:name", default="", namespaces=ns) or "")
+            for a in entry.findall("atom:author", ns)
+        ]
         pdf_url = ""
         for link in entry.findall("atom:link", ns):
             if link.attrib.get("title") == "pdf":
                 pdf_url = link.attrib.get("href", "")
                 break
+
         papers.append({
             "id": paper_id or title,
             "title": title,
@@ -314,17 +339,17 @@ def search_arxiv(query, max_results=8):
             "venue": "arXiv",
             "year": published[:4] if published else "",
             "source": "arxiv",
-            "score": 0.76,
+            "score": 0.75,
             "open_access": True,
-            "external_ids": {"arxiv": (paper_id or "").split("/")[-1]},
         })
+
     return papers
 
 
-def search_crossref(query, mode="topic", max_results=8):
-    headers = {"User-Agent": "dvnc-ai-space/0.3"}
+def search_crossref(query: str, mode: str = "topic", max_results: int = 6) -> List[Dict[str, Any]]:
+    headers = {"User-Agent": "dvnc-ai-space/1.0"}
     if mode == "doi":
-        url = f"https://api.crossref.org/works/{urllib.parse.quote(query)}"
+        url = f"https://api.crossref.org/works/{urllib.parse.quote(normalize_doi(query))}"
         response = requests.get(url, headers=headers, timeout=REQUEST_TIMEOUT)
         if response.status_code != 200:
             return []
@@ -338,135 +363,727 @@ def search_crossref(query, mode="topic", max_results=8):
         response = requests.get("https://api.crossref.org/works", params=params, headers=headers, timeout=REQUEST_TIMEOUT)
         response.raise_for_status()
         items = response.json().get("message", {}).get("items", [])
+
     out = []
     for item in items:
         authors = []
         for a in item.get("author", []) or []:
             name = " ".join(filter(None, [a.get("given"), a.get("family")])).strip()
             if name:
-                authors.append(name)
-        title = (item.get("title") or ["Untitled"])[0]
+                authors.append(clean_pdf_text(name))
+
+        title = clean_pdf_text((item.get("title") or ["Untitled"])[0])
+        abstract = clean_pdf_text(re.sub("<.*?>", " ", item.get("abstract") or ""))
         year = ""
         for key in ["published-print", "published-online", "created"]:
             if item.get(key, {}).get("date-parts"):
                 year = str(item[key]["date-parts"][0][0])
                 break
-        abstract = re.sub("<.*?>", "", item.get("abstract") or "")
-        doi = normalize_doi(item.get("DOI", ""))
+
         out.append({
-            "id": doi or title,
-            "title": norm_text(title),
-            "summary": norm_text(abstract)[:500],
-            "abstract": norm_text(abstract),
+            "id": normalize_doi(item.get("DOI", "")) or title,
+            "title": title,
+            "summary": truncate_text(abstract, 420),
+            "abstract": abstract,
             "published": year,
             "authors": authors,
             "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
             "url": item.get("URL", ""),
             "pdf": "",
-            "doi": doi,
-            "venue": (item.get("container-title") or [""])[0],
+            "doi": normalize_doi(item.get("DOI", "")),
+            "venue": clean_pdf_text((item.get("container-title") or [""])[0]),
             "year": year,
             "source": "crossref",
             "score": 0.72,
             "open_access": None,
-            "external_ids": {"crossref": doi} if doi else {},
         })
+
     return out
 
 
-def search_openalex(query, mode="topic", max_results=8):
-    params = {"per-page": max_results}
-    if mode == "doi":
-        doi = normalize_doi(query)
-        params["filter"] = f"doi:https://doi.org/{doi}"
-    else:
-        params["search"] = query
-    response = requests.get("https://api.openalex.org/works", params=params, timeout=REQUEST_TIMEOUT)
-    response.raise_for_status()
-    items = response.json().get("results", [])
-    out = []
-    for item in items:
-        authors = []
-        for auth in item.get("authorships", [])[:8]:
-            author = auth.get("author") or {}
-            if author.get("display_name"):
-                authors.append(author["display_name"])
-        oa = item.get("open_access") or {}
-        doi = normalize_doi(item.get("doi") or "")
-        abstract = parse_openalex_abstract(item.get("abstract_inverted_index"))
-        out.append({
-            "id": item.get("id") or doi or item.get("title"),
-            "title": norm_text(item.get("title")),
-            "summary": norm_text(abstract)[:500],
-            "abstract": norm_text(abstract),
-            "published": str(item.get("publication_year") or ""),
-            "authors": authors,
-            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
-            "url": (item.get("primary_location") or {}).get("landing_page_url") or item.get("id") or "",
-            "pdf": oa.get("oa_url") or "",
-            "doi": doi,
-            "venue": ((item.get("primary_location") or {}).get("source") or {}).get("display_name") or "",
-            "year": str(item.get("publication_year") or ""),
-            "source": "openalex",
-            "score": 0.80,
-            "open_access": oa.get("is_oa"),
-            "external_ids": item.get("ids") or {},
+def discover_papers(query: str, search_mode: str, sources: List[str], max_results: int = GRAPH_MAX_RESULTS) -> List[Dict[str, Any]]:
+    query = norm_text(query)
+    if not query:
+        return []
+
+    mode = detect_query_type(query) if search_mode == "autonomous_web" else search_mode
+    items = []
+
+    if mode == "link":
+        doi_match = re.search(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", query, flags=re.I)
+        if doi_match:
+            mode = "doi"
+            query = doi_match.group(0)
+        else:
+            return [{
+                "id": query,
+                "title": query,
+                "summary": "Direct web link supplied.",
+                "abstract": "",
+                "published": "",
+                "authors": [],
+                "authors_text": "Unknown authors",
+                "url": query,
+                "pdf": query if query.lower().endswith(".pdf") else "",
+                "doi": "",
+                "venue": "Web link",
+                "year": "",
+                "source": "link",
+                "score": 0.4,
+                "open_access": None,
+                "concepts": [],
+                "claims": [],
+                "learned_score": 0.4,
+            }]
+
+    selected = ensure_list(sources) or DEFAULT_SOURCES
+
+    if "arxiv" in selected and mode != "doi":
+        try:
+            items.extend(search_arxiv(query, min(max_results, 6)))
+        except Exception:
+            pass
+
+    if "crossref" in selected:
+        try:
+            items.extend(search_crossref(query, mode=mode, max_results=min(max_results, 8)))
+        except Exception:
+            pass
+
+    enriched = [enrich_paper(query, item) for item in items]
+    enriched = dedupe_papers(enriched)
+    return enriched[:max_results]
+
+
+def build_journal_html(query: str) -> str:
+    q = urllib.parse.quote_plus(query or "biomaterials cardiac repair")
+    cards = []
+
+    for journal in JOURNALS:
+        if "ieeexplore" in journal["url"]:
+            url = f"https://ieeexplore.ieee.org/search/searchresult.jsp?queryText={q}"
+        elif "?" in journal["url"]:
+            url = f"{journal['url']}&q={q}"
+        else:
+            url = f"{journal['url']}?q={q}"
+
+        cards.append(
+            f"""
+            <a class="journal-card" href="{safe_text(url)}" target="_blank" rel="noopener noreferrer">
+              <div>
+                <h4>{safe_text(journal['name'])}</h4>
+                <p>{safe_text(journal['desc'])}</p>
+              </div>
+              <span>Open</span>
+            </a>
+            """
+        )
+
+    return """
+    <style>
+      .journal-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(240px,1fr));gap:12px}
+      .journal-card{display:flex;justify-content:space-between;gap:12px;padding:14px 16px;border-radius:14px;
+        border:1px solid #dbe3f0;background:#f8fbff;text-decoration:none;color:#0f172a}
+      .journal-card h4{margin:0 0 4px;font-size:15px}
+      .journal-card p{margin:0;color:#475569;font-size:13px;line-height:1.45}
+      .journal-card span{align-self:center;font-size:12px;font-weight:700;color:#2563eb}
+    </style>
+    <div class="journal-grid">""" + "".join(cards) + "</div>"
+
+
+def render_papers_html(papers: List[Dict[str, Any]]) -> str:
+    if not papers:
+        return '<div style="padding:18px;border:1px solid #e2e8f0;border-radius:16px;background:#fff">No papers found yet.</div>'
+
+    cards = []
+    for i, paper in enumerate(papers, start=1):
+        concepts = ", ".join((paper.get("concepts") or [])[:5]) or "None extracted"
+        cards.append(
+            f"""
+            <article style="border:1px solid #e2e8f0;border-radius:16px;padding:16px;background:#fff;display:flex;flex-direction:column;gap:10px">
+              <div style="display:flex;gap:8px;flex-wrap:wrap">
+                <span style="padding:4px 8px;border-radius:999px;background:#eff6ff;color:#1d4ed8;font-size:12px;font-weight:700">{safe_text(paper.get('source','paper'))}</span>
+                <span style="padding:4px 8px;border-radius:999px;background:#f8fafc;color:#475569;font-size:12px">{safe_text(paper.get('year',''))}</span>
+              </div>
+              <h4 style="margin:0;font-size:16px;line-height:1.35">{i}. {safe_text(paper.get('title','Untitled'))}</h4>
+              <p style="margin:0;color:#334155;line-height:1.55">{safe_text(paper.get('summary','No abstract snippet available.'))}</p>
+              <div style="font-size:13px;color:#475569;line-height:1.6">
+                <div><strong>Authors:</strong> {safe_text(paper.get('authors_text','Unknown authors'))}</div>
+                <div><strong>Venue:</strong> {safe_text(paper.get('venue','Unknown venue'))}</div>
+                <div><strong>Score:</strong> {safe_text(paper.get('learned_score', paper.get('score', 0)))}</div>
+                <div><strong>Concepts:</strong> {safe_text(concepts)}</div>
+              </div>
+            </article>
+            """
+        )
+
+    return '<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(280px,1fr));gap:14px">' + "".join(cards) + "</div>"
+
+
+def uploaded_pdf_summary(file_obj) -> str:
+    if not file_obj:
+        return "No PDF uploaded yet."
+    path = getattr(file_obj, "name", None) or str(file_obj)
+    return f"Uploaded PDF ready: {Path(path).name}"
+
+
+def _graph_kind_style(kind: str) -> Dict[str, Any]:
+    return {
+        "query": {"color": "#1f8ef1", "size": 14},
+        "paper": {"color": "#00c49a", "size": 10},
+        "upload": {"color": "#ff9f43", "size": 11},
+        "concept": {"color": "#a66cff", "size": 8},
+        "author": {"color": "#f368e0", "size": 7},
+        "claim": {"color": "#ff6b6b", "size": 8},
+        "reference": {"color": "#6b7280", "size": 7},
+    }.get(kind, {"color": "#94a3b8", "size": 7})
+
+
+def build_learning_graph_html(nodes, edges, title="Self-Learning Knowledge Graph"):
+    nodes = ensure_list(nodes)
+    edges = ensure_list(edges)
+
+    if not nodes:
+        return """
+        <div style="padding:18px;border:1px solid #dbe3f0;border-radius:18px;background:#fff;overflow:auto">
+          <h3 style="margin:0 0 8px">Self-Learning Knowledge Graph</h3>
+          <p style="margin:0;color:#475569">No papers mapped yet.</p>
+        </div>
+        """
+
+    graph_nodes = []
+    for node in nodes:
+        kind = (node.get("kind") or node.get("type") or "paper").lower()
+        if kind == "topic":
+            kind = "query"
+        if kind == "uploadedpdf":
+            kind = "upload"
+        style = _graph_kind_style(kind)
+        graph_nodes.append({
+            "id": node.get("id"),
+            "label": truncate_text(node.get("label") or node.get("title") or node.get("id") or "node", 100),
+            "kind": kind,
+            "color": style["color"],
+            "val": style["size"],
+            "detail": {
+                "type": kind,
+                "title": node.get("title") or node.get("label") or "",
+                "source": node.get("source") or "",
+                "year": node.get("year") or "",
+                "doi": node.get("doi") or "",
+                "authors_text": node.get("authors_text") or "",
+                "text": node.get("text") or "",
+            },
         })
-    return out
 
+    graph_links = []
+    for edge in edges:
+        graph_links.append({
+            "source": edge.get("source"),
+            "target": edge.get("target"),
+            "label": edge.get("type", "RELATES_TO"),
+            "type": edge.get("type", "RELATES_TO"),
+        })
 
-def search_semantic_scholar(query, mode="topic", max_results=8):
-    headers = {}
-    if SEMANTIC_SCHOLAR_API_KEY:
-        headers["x-api-key"] = SEMANTIC_SCHOLAR_API_KEY
-    fields = "title,authors,year,abstract,venue,externalIds,url,openAccessPdf"
-    if mode == "doi":
-        doi = normalize_doi(query)
-        url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{urllib.parse.quote(doi)}"
-        response = requests.get(url, params={"fields": fields}, headers=headers, timeout=REQUEST_TIMEOUT)
-        if response.status_code != 200:
-            return []
-        items = [response.json()]
-    else:
-        response = requests.get(
-            "https://api.semanticscholar.org/graph/v1/paper/search",
-            params={"query": query, "limit": max_results, "fields": fields},
-            headers=headers,
-            timeout=REQUEST_TIMEOUT,
+    payload = json.dumps({"nodes": graph_nodes, "links": graph_links, "title": title}, ensure_ascii=False)
+
+    iframe_doc = f"""
+<!doctype html>
+<html>
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<style>
+html,body{{margin:0;height:100%;overflow:hidden;background:#0b1020;color:#eef2ff;font-family:Inter,Arial,sans-serif}}
+#graph{{position:absolute;inset:0}}
+.panel{{position:absolute;left:14px;top:14px;z-index:10;max-width:430px;padding:12px 14px;border-radius:14px;
+background:rgba(8,12,24,.76);border:1px solid rgba(255,255,255,.12);backdrop-filter:blur(12px)}}
+.panel h3{{margin:0 0 6px;font-size:17px}}
+.panel p{{margin:0;font-size:12px;line-height:1.5;color:#cbd5e1}}
+.details{{position:absolute;right:14px;top:14px;z-index:10;width:min(320px,calc(100% - 28px));max-height:calc(100% - 28px);overflow:auto;
+padding:12px 14px;border-radius:14px;background:rgba(8,12,24,.76);border:1px solid rgba(255,255,255,.12);backdrop-filter:blur(12px)}}
+.details h4{{margin:0 0 8px;font-size:14px}}
+.details dl{{display:grid;grid-template-columns:auto 1fr;gap:6px 10px;margin:0;font-size:12px}}
+.details dt{{color:#93c5fd}} .details dd{{margin:0;color:#e2e8f0;word-break:break-word}}
+.hint{{position:absolute;left:14px;bottom:14px;z-index:10;padding:9px 12px;border-radius:12px;
+font-size:12px;background:rgba(8,12,24,.76);border:1px solid rgba(255,255,255,.10);color:#dbeafe}}
+</style>
+<script src="https://unpkg.com/three@0.160.0/build/three.min.js"></script>
+<script src="https://unpkg.com/3d-force-graph"></script>
+</head>
+<body>
+<div id="graph"></div>
+<div class="panel">
+  <h3>{safe_text(title)}</h3>
+  <p>Drag background to orbit, scroll to zoom, right-drag to pan, and drag a node to move and pin it in 3D space.</p>
+</div>
+<div class="details" id="details">
+  <h4>Node details</h4>
+  <p style="margin:0;color:#cbd5e1;font-size:12px">Click a node to inspect it.</p>
+</div>
+<div class="hint">3D graph enabled: orbit, zoom, pan, drag nodes.</div>
+<script>
+const data = {payload};
+
+const Graph = ForceGraph3D()(document.getElementById('graph'))
+  .backgroundColor('#00000000')
+  .graphData(data)
+  .nodeRelSize(6)
+  .nodeLabel(node => `<div style="padding:6px 8px"><strong>${{node.label}}</strong><br/>${{node.kind}}</div>`)
+  .linkLabel(link => link.label)
+  .linkWidth(link => ['ABOUT','UPLOADED_SOURCE'].includes(link.type) ? 2.4 : 1.2)
+  .linkDirectionalParticles(link => ['ABOUT','UPLOADED_SOURCE'].includes(link.type) ? 2 : 0)
+  .linkDirectionalParticleWidth(2)
+  .nodeColor(node => node.color)
+  .nodeVal(node => node.val)
+  .onNodeClick(node => {{
+    const d = node.detail || {{}};
+    document.getElementById('details').innerHTML = `
+      <h4>Node details</h4>
+      <dl>
+        <dt>Label</dt><dd>${{node.label || ''}}</dd>
+        <dt>Type</dt><dd>${{d.type || node.kind || ''}}</dd>
+        <dt>Title</dt><dd>${{d.title || '—'}}</dd>
+        <dt>Source</dt><dd>${{d.source || '—'}}</dd>
+        <dt>Year</dt><dd>${{d.year || '—'}}</dd>
+        <dt>DOI</dt><dd>${{d.doi || '—'}}</dd>
+        <dt>Authors</dt><dd>${{d.authors_text || '—'}}</dd>
+        <dt>Text</dt><dd>${{d.text || '—'}}</dd>
+      </dl>`;
+  }})
+  .onNodeDragEnd(node => {{
+    node.fx = node.x;
+    node.fy = node.y;
+    node.fz = node.z;
+  }});
+
+Graph.cameraPosition({{ z: 180 }});
+</script>
+</body>
+</html>
+"""
+    return f"""
+    <div style="border:1px solid #dbe3f0;border-radius:18px;background:#fff;overflow:auto">
+      <iframe
+        title="{safe_text(title)}"
+        srcdoc="{html.escape(iframe_doc, quote=True)}"
+        style="width:100%;height:760px;border:0;border-radius:18px;overflow:auto;background:#0b1020"
+        sandbox="allow-scripts allow-same-origin"
+      ></iframe>
+    </div>
+    """
+
+
+def seed_graph_state(query: str, papers: List[Dict[str, Any]], uploaded_name: Optional[str] = None):
+    nodes = [{"id": "query", "label": query or "Research topic", "kind": "query"}]
+    edges = []
+
+    for i, paper in enumerate(papers[:6], start=1):
+        pid = f"paper_{i}"
+        nodes.append({
+            "id": pid,
+            "label": paper.get("title", f"Paper {i}"),
+            "kind": "paper",
+            "title": paper.get("title"),
+            "year": paper.get("year"),
+            "source": paper.get("source"),
+            "authors_text": paper.get("authors_text"),
+        })
+        edges.append({"source": "query", "target": pid, "type": "ABOUT"})
+
+        for concept in (paper.get("concepts") or [])[:3]:
+            cid = f"concept_{i}_{slugify(concept)[:30]}"
+            nodes.append({"id": cid, "label": concept, "kind": "concept"})
+            edges.append({"source": pid, "target": cid, "type": "MENTIONS"})
+
+    if uploaded_name:
+        nodes.append({"id": "upload", "label": uploaded_name, "kind": "upload"})
+        edges.append({"source": "query", "target": "upload", "type": "UPLOADED_SOURCE"})
+
+    return nodes, edges
+
+
+def paper_choice_value(index: int, paper: Dict[str, Any]) -> str:
+    return f"{index}|{slugify(paper.get('title', 'paper'))[:40]}"
+
+
+def paper_choice_label(index: int, paper: Dict[str, Any]) -> str:
+    title = paper.get("title", "Untitled")
+    authors = paper.get("authors_text", "Unknown authors")
+    score = paper.get("learned_score", paper.get("score", 0))
+    return f"[{paper.get('source','paper')}] {title} — {authors} — score {score}"
+
+
+def format_selection_choices(papers: List[Dict[str, Any]]):
+    return [(paper_choice_label(i, paper), paper_choice_value(i, paper)) for i, paper in enumerate(papers)]
+
+
+def resolve_selected_papers(selected_indices, papers_state):
+    papers = ensure_list(papers_state)
+    selected_indices = ensure_list(selected_indices)
+    selected = []
+
+    for item in selected_indices:
+        if isinstance(item, int) and 0 <= item < len(papers):
+            selected.append(papers[item])
+            continue
+        item = str(item)
+        if item.isdigit():
+            idx = int(item)
+            if 0 <= idx < len(papers):
+                selected.append(papers[idx])
+                continue
+        if "|" in item and item.split("|", 1)[0].isdigit():
+            idx = int(item.split("|", 1)[0])
+            if 0 <= idx < len(papers):
+                selected.append(papers[idx])
+
+    deduped = []
+    seen = set()
+    for paper in selected:
+        key = paper_identity_key(paper)
+        if key not in seen:
+            seen.add(key)
+            deduped.append(paper)
+    return deduped
+
+
+def parse_uploaded_pdf(file_obj, parser_order=None):
+    if not file_obj:
+        return "### PDF parse status\n\n- No PDF uploaded yet.", {}
+
+    path = getattr(file_obj, "name", None) or str(file_obj)
+    file_name = Path(path).name
+
+    if fitz is None:
+        parsed = {
+            "parser": "none",
+            "title": file_name,
+            "abstract": "",
+            "authors": [],
+            "sections": [],
+            "references": [],
+            "concepts": [],
+            "claims": [],
+            "raw_text": "",
+        }
+        return "### PDF parse status\n\n- PyMuPDF is unavailable in this runtime.", parsed
+
+    try:
+        doc = fitz.open(path)
+        text = "\n".join(page.get_text("text") for page in doc[: min(len(doc), 8)])
+        text = clean_pdf_text(text)
+    except Exception as e:
+        return f"### PDF parse status\n\n- Failed to parse PDF\n- Error: {safe_text(str(e))}", {}
+
+    title = extract_title_from_text(text, fallback=file_name)
+
+    abstract = ""
+    match = re.search(r"abstract\s*(.+?)(?:\n\s*\n|\n(?:1|i)[\.\s]|\nintroduction)", text, re.I | re.S)
+    if match:
+        abstract = truncate_text(clean_pdf_text(match.group(1)), 2400)
+
+    sections = []
+    blocks = re.split(r"\n(?=[A-Z][A-Z0-9 \-]{3,80}\n)", text)
+    for block in blocks[:10]:
+        lines = [norm_text(x) for x in block.splitlines() if norm_text(x)]
+        if not lines:
+            continue
+        heading = lines[0][:80]
+        body = " ".join(lines[1:] if len(lines) > 1 else [])
+        if len(body) > 80:
+            sections.append({"heading": heading, "text": truncate_text(body, 1800)})
+
+    parsed = {
+        "parser": "pymupdf",
+        "title": title,
+        "abstract": abstract,
+        "authors": [],
+        "sections": sections,
+        "references": extract_references_from_text(text),
+        "concepts": extract_concepts_from_text(f"{title} {abstract} {text[:16000]}", GRAPH_MAX_CONCEPTS),
+        "claims": extract_claim_like_sentences(text, GRAPH_MAX_CLAIMS),
+        "raw_text": text[:30000],
+    }
+
+    summary = "\n".join([
+        "### PDF parse status",
+        "",
+        f"- Parser used: {parsed['parser']}",
+        f"- Title: {parsed['title']}",
+        f"- Abstract found: {'Yes' if parsed['abstract'] else 'No'}",
+        f"- Sections extracted: {len(parsed['sections'])}",
+        f"- References extracted: {len(parsed['references'])}",
+        f"- Concepts extracted: {len(parsed['concepts'])}",
+        f"- Claims extracted: {len(parsed['claims'])}",
+    ])
+    return summary, parsed
+
+
+def render_parse_result(parsed):
+    if not parsed or not isinstance(parsed, dict) or not parsed.get("title"):
+        return '<div style="padding:18px;border:1px solid #e2e8f0;border-radius:16px;background:#fff">No parsed document yet.</div>'
+
+    sections_html = []
+    for section in parsed.get("sections", [])[:8]:
+        sections_html.append(
+            f"""
+            <details style="border:1px solid #e2e8f0;border-radius:14px;padding:12px 14px;background:#fff">
+              <summary style="cursor:pointer;font-weight:700">{safe_text(section.get('heading','Section'))}</summary>
+              <p style="margin:10px 0 0;color:#334155;line-height:1.6">{safe_text(section.get('text',''))}</p>
+            </details>
+            """
         )
-        if response.status_code != 200:
-            return []
-        items = response.json().get("data", [])
-    out = []
-    for item in items:
-        external = item.get("externalIds") or {}
-        authors = [a.get("name") for a in item.get("authors", []) if a.get("name")]
-        out.append({
-            "id": external.get("CorpusId") or external.get("DOI") or item.get("title"),
-            "title": norm_text(item.get("title")),
-            "summary": norm_text(item.get("abstract", ""))[:500],
-            "abstract": norm_text(item.get("abstract", "")),
-            "published": str(item.get("year") or ""),
-            "authors": authors,
-            "authors_text": ", ".join(authors[:4]) if authors else "Unknown authors",
-            "url": item.get("url") or "",
-            "pdf": (item.get("openAccessPdf") or {}).get("url") or "",
-            "doi": normalize_doi(external.get("DOI", "")),
-            "venue": item.get("venue") or "",
-            "year": str(item.get("year") or ""),
-            "source": "semantic_scholar",
-            "score": 0.84,
-            "open_access": bool((item.get("openAccessPdf") or {}).get("url")),
-            "external_ids": external,
+
+    concepts = "".join(f"<li>{safe_text(x)}</li>" for x in parsed.get("concepts", [])[:12]) or "<li>None</li>"
+    claims = "".join(f"<li>{safe_text(x)}</li>" for x in parsed.get("claims", [])[:8]) or "<li>None</li>"
+    refs = "".join(
+        f"<li>{safe_text(r.get('title','Untitled'))}{' · DOI ' + safe_text(r.get('doi')) if r.get('doi') else ''}</li>"
+        for r in parsed.get("references", [])[:12]
+    ) or "<li>None</li>"
+
+    return f"""
+    <div style="display:grid;gap:14px">
+      <div style="padding:18px;border:1px solid #e2e8f0;border-radius:16px;background:#fff">
+        <h3 style="margin:0 0 8px">{safe_text(parsed.get('title','Parsed document'))}</h3>
+        <p style="margin:0;color:#475569;line-height:1.6">{safe_text(parsed.get('abstract','No abstract extracted.'))}</p>
+      </div>
+      <div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:14px">
+        <div style="padding:16px;border:1px solid #e2e8f0;border-radius:16px;background:#fff"><h4>Concepts</h4><ul>{concepts}</ul></div>
+        <div style="padding:16px;border:1px solid #e2e8f0;border-radius:16px;background:#fff"><h4>Claims</h4><ul>{claims}</ul></div>
+        <div style="padding:16px;border:1px solid #e2e8f0;border-radius:16px;background:#fff"><h4>References</h4><ul>{refs}</ul></div>
+      </div>
+      <div style="display:grid;gap:12px">{''.join(sections_html) if sections_html else ''}</div>
+    </div>
+    """
+
+
+def build_ingest_payload(query: str, selected_papers: List[Dict[str, Any]], parsed_pdf: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+    nodes = []
+    edges = []
+
+    def add_node(node_id: str, node_type: str, label: str, **attrs):
+        node = {"id": node_id, "type": node_type, "label": label}
+        node.update(attrs)
+        nodes.append(node)
+
+    def add_edge(source: str, target: str, edge_type: str):
+        edges.append({"source": source, "target": target, "type": edge_type})
+
+    topic_id = "topic:query"
+    add_node(topic_id, "Topic", query or "Research topic", query=query)
+
+    for i, paper in enumerate(selected_papers, start=1):
+        paper_id = normalize_doi(paper.get("doi")) or f"paper:{i}:{slugify(paper.get('title','paper'))[:40]}"
+        add_node(
+            paper_id,
+            "Paper",
+            paper.get("title", f"Paper {i}"),
+            title=paper.get("title"),
+            year=paper.get("year"),
+            venue=paper.get("venue"),
+            doi=paper.get("doi"),
+            source=paper.get("source"),
+            authors_text=paper.get("authors_text"),
+        )
+        add_edge(topic_id, paper_id, "ABOUT")
+
+        for author in paper.get("authors", [])[:4]:
+            author_id = f"author:{slugify(author)[:40]}"
+            add_node(author_id, "Author", author, name=author)
+            add_edge(paper_id, author_id, "WRITTEN_BY")
+
+        for concept in (paper.get("concepts") or [])[:6]:
+            concept_id = f"concept:{slugify(concept)[:48]}"
+            add_node(concept_id, "Concept", concept, name=concept)
+            add_edge(paper_id, concept_id, "MENTIONS")
+
+        for claim in (paper.get("claims") or [])[:3]:
+            claim_id = f"claim:{slugify(claim)[:48]}"
+            add_node(claim_id, "Claim", truncate_text(claim, 120), text=claim)
+            add_edge(paper_id, claim_id, "ASSERTS")
+
+    if parsed_pdf and parsed_pdf.get("title"):
+        upload_id = "upload:pdf"
+        add_node(upload_id, "UploadedPDF", parsed_pdf.get("title"), parser=parsed_pdf.get("parser"))
+        add_edge(topic_id, upload_id, "UPLOADED_SOURCE")
+
+        for concept in (parsed_pdf.get("concepts") or [])[:6]:
+            concept_id = f"concept:{slugify(concept)[:48]}"
+            add_node(concept_id, "Concept", concept, name=concept)
+            add_edge(upload_id, concept_id, "MENTIONS")
+
+        for claim in (parsed_pdf.get("claims") or [])[:4]:
+            claim_id = f"claim:{slugify(claim)[:48]}"
+            add_node(claim_id, "Claim", truncate_text(claim, 120), text=claim)
+            add_edge(upload_id, claim_id, "ASSERTS")
+
+        for idx, ref in enumerate(parsed_pdf.get("references", [])[:12], start=1):
+            rid = normalize_doi(ref.get("doi")) or f"ref:{idx}:{slugify(ref.get('title','reference'))[:40]}"
+            add_node(rid, "Reference", ref.get("title", f"Reference {idx}"), doi=ref.get("doi", ""))
+            add_edge(upload_id, rid, "CITES")
+
+    dedup_nodes = {}
+    for node in nodes:
+        dedup_nodes[node["id"]] = node
+
+    return {"status": "ok", "nodes": list(dedup_nodes.values()), "edges": edges}
+
+
+def ingest_selected_papers(query, selected_indices, papers_state, pdf_file, parsed_state):
+    papers = ensure_list(papers_state)
+    selected = resolve_selected_papers(selected_indices, papers)
+
+    if not selected and papers:
+        selected = papers[:3]
+
+    parsed = parsed_state if isinstance(parsed_state, dict) else {}
+    query_text = norm_text(query or "") or parsed.get("title") or "Research topic"
+
+    if not selected and not parsed.get("title"):
+        empty_payload = {"status": "empty", "nodes": [], "edges": []}
+        graph_html = build_learning_graph_html([], [], "Self-Learning Knowledge Graph")
+        status_md = "### Graph ingest status\n\n- No papers selected.\n- No parsed PDF available."
+        return graph_html, status_md, empty_payload
+
+    payload = build_ingest_payload(query_text, selected, parsed)
+
+    GRAPH_MEMORY["queries"].append(query_text)
+    GRAPH_MEMORY["nodes"] = {node["id"]: node for node in payload["nodes"]}
+    GRAPH_MEMORY["edges"] = payload["edges"]
+    for node in payload["nodes"]:
+        if node.get("type") == "Paper":
+            GRAPH_MEMORY["papers"][node["id"]] = node
+
+    graph_nodes = []
+    for node in payload["nodes"]:
+        kind = node.get("type", "Paper")
+        if kind == "Topic":
+            kind = "query"
+        elif kind == "Paper":
+            kind = "paper"
+        elif kind == "UploadedPDF":
+            kind = "upload"
+        elif kind == "Concept":
+            kind = "concept"
+        elif kind == "Author":
+            kind = "author"
+        elif kind == "Claim":
+            kind = "claim"
+        elif kind == "Reference":
+            kind = "reference"
+        else:
+            kind = "paper"
+
+        graph_nodes.append({
+            "id": node["id"],
+            "label": node.get("label", node["id"]),
+            "kind": kind,
+            "title": node.get("title", node.get("label", "")),
+            "year": node.get("year", ""),
+            "doi": node.get("doi", ""),
+            "source": node.get("source", ""),
+            "authors_text": node.get("authors_text", ""),
+            "text": node.get("text", ""),
         })
-    return out
 
+    graph_html = build_learning_graph_html(graph_nodes, payload["edges"], "Selected Research Graph")
 
-def search_europe_pmc(query, mode="topic", max_results=8):
-    epmc_query = f'DOI:"{query}"' if mode == "doi" else query
-    params = {"query": epmc_query, "format": "json", "pageSize": max_results, "resultType": "core"}
-    response = requests.get("https://www.ebi.ac.uk/europepmc/webservices/rest/search", params=params, timeout=REQUEST_TIMEOUT)
-    if response.status_code != 200:
-        return []
-    items = response.json().get("resultList", {}).get("result", [])
-    
+    node_lines = [f"- [{n.get('type')}] {n.get('label')}" for n in payload["nodes"][:24]]
+    edge_lines = [f"- {e.get('source')} --{e.get('type')}--> {e.get('target')}" for e in payload["edges"][:30]]
+
+    status_md = "\n".join([
+        "### Graph ingest status",
+        "",
+        f"- Topic: {query_text}",
+        f"- Selected papers ingested: {len(selected)}",
+        f"- Uploaded PDF parsed: {'Yes' if parsed.get('title') else 'No'}",
+        f"- Total nodes created: {len(payload['nodes'])}",
+        f"- Total edges created: {len(payload['edges'])}",
+        "",
+        "### Nodes",
+        *(node_lines or ["- None"]),
+        "",
+        "### Edges",
+        *(edge_lines or ["- None"]),
+    ])
+
+    return graph_html, status_md, payload
+
+
+def run_paper_discovery(query, search_mode, sources, pdf_file):
+    query_text = norm_text(query or "")
+    chosen_sources = ensure_list(sources) or DEFAULT_SOURCES
+    uploaded_name = Path(getattr(pdf_file, "name", str(pdf_file))).name if pdf_file else None
+
+    if not query_text and not pdf_file:
+        empty_graph = build_learning_graph_html([], [], "Self-Learning Knowledge Graph")
+        return (
+            empty_graph,
+            '<div style="padding:18px;border:1px solid #e2e8f0;border-radius:16px;background:#fff">Enter a query or upload a PDF to begin.</div>',
+            build_journal_html("biomaterials cardiac repair"),
+            "No PDF uploaded yet.",
+            gr.update(choices=[], value=[]),
+            [],
+            "### No discovery results yet.",
+        )
+
+    if not query_text and pdf_file:
+        nodes, edges = seed_graph_state("", [], uploaded_name)
+        return (
+            build_learning_graph_html(nodes, edges, "Uploaded PDF Waiting for Parse"),
+            '<div style="padding:18px;border:1px solid #e2e8f0;border-radius:16px;background:#fff">Upload detected. Parse the PDF or enter a topic.</div>',
+            build_journal_html("biomaterials cardiac repair"),
+            uploaded_pdf_summary(pdf_file),
+            gr.update(choices=[], value=[]),
+            [],
+            "### Upload detected.\n\n- Parse the PDF to extract concepts, claims, and references.",
+        )
+
+    try:
+        papers = discover_papers(query_text, search_mode, chosen_sources, GRAPH_MAX_RESULTS)
+    except Exception as e:
+        papers = []
+        error_message = str(e)
+    else:
+        error_message = ""
+
+    nodes, edges = seed_graph_state(query_text, papers[:6], uploaded_name)
+    graph_html = build_learning_graph_html(nodes, edges, "Self-Learning Knowledge Graph")
+    papers_html = render_papers_html(papers) if not error_message else f'<div style="padding:18px;border:1px solid #fecaca;border-radius:16px;background:#fff1f2">Paper search failed: {safe_text(error_message)}</div>'
+    journals_html = build_journal_html(query_text or "biomaterials cardiac repair")
+    pdf_summary = uploaded_pdf_summary(pdf_file)
+    choices = format_selection_choices(papers)
+
+    status_md = "\n".join([
+        "### Discovery results",
+        "",
+        f"- Query: {query_text}",
+        f"- Sources: {', '.join(chosen_sources)}",
+        f"- Candidates found: {len(papers)}",
+        "- Select papers below, or ingest the top papers automatically.",
+    ])
+
+    return (
+        graph_html,
+        papers_html,
+        journals_html,
+        pdf_summary,
+        gr.update(choices=choices, value=[]),
+        papers,
+        status_md,
+    )
+
+
+__all__ = [
+    "DEFAULT_SOURCES",
+    "SEARCH_MODES",
+    "SOURCE_OPTIONS",
+    "build_learning_graph_html",
+    "build_journal_html",
+    "ingest_selected_papers",
+    "parse_uploaded_pdf",
+    "render_parse_result",
+    "run_paper_discovery",
+    "safe_text",
+] html