Merge pull request #10 from KevinIsInCoding/feat/automated-seed-entities

config.py

@@ -22,12 +22,27 @@ GRAPH_JSON_PATH = DATA_DIR / "graph" / "als_graph.json"
 CHROMA_DIR = DATA_DIR / "chroma"
 CHROMA_COLLECTION = "als_papers"
 
+# Seed entity files
+MANUAL_SEEDS_PATH = DATA_DIR / "seeds" / "manual_seeds.json"
+DERIVED_SEEDS_PATH = DATA_DIR / "seeds" / "derived_seeds.json"
+SEED_PROMOTION_THRESHOLD = 5   # min papers for entity to become a derived seed
+REFRESH_STATE_PATH = DATA_DIR / ".refresh_state.json"
+
 # PubMed ingestion defaults
+PUBMED_BASE_QUERY = (
+    '"amyotrophic lateral sclerosis"[MeSH Major Topic] '
+    "AND hasabstract[text]"
+)
 PUBMED_DEFAULT_QUERY = (
     '"amyotrophic lateral sclerosis"[MeSH Major Topic] '
     'AND ("2018"[PDAT]:"2024"[PDAT]) '
     "AND hasabstract[text]"
 )
+PUBMED_REFRESH_QUERY_TEMPLATE = (
+    '"amyotrophic lateral sclerosis"[MeSH Major Topic] '
+    'AND ("{since_date}"[PDAT]:"3000"[PDAT]) '
+    "AND hasabstract[text]"
+)
 PUBMED_DEFAULT_MAX = 500
 PUBMED_BATCH_SIZE = 200  # PMIDs per Entrez efetch call
 
@@ -53,34 +68,6 @@ CROSS_ENCODER_TOP_N = 15  # final papers sent to Claude for synthesis
 KG_EXPANSION_HOPS = 1  # hops for query entity expansion
 KG_MIN_EDGE_CONFIDENCE = 0.3  # edges below this are excluded from traversal
 
-# ALS seed entities — pre-populate the graph before paper-derived extraction
-ALS_SEED_ENTITIES = {
-    "genes": [
-        "SOD1", "TARDBP", "FUS", "C9orf72", "ATXN2",
-        "TBK1", "OPTN", "UBQLN2", "VCP", "NEK1",
-        "ANG", "SETX", "SIGMAR1", "CHCHD10", "MATR3",
-    ],
-    "proteins": [
-        "TDP-43", "FUS protein", "SOD1 protein", "Alsin",
-        "Optineurin", "p62", "Ubiquilin-2",
-    ],
-    "compounds": [
-        "riluzole", "edaravone", "tofersen", "AMX0035",
-        "mexiletine", "masitinib", "bosutinib",
-    ],
-    "mechanisms": [
-        "glutamate excitotoxicity", "oxidative stress",
-        "neuroinflammation", "protein aggregation",
-        "RNA metabolism dysfunction", "mitochondrial dysfunction",
-        "axonal transport defect", "autophagy impairment",
-    ],
-    "phenotypes": [
-        "upper motor neuron degeneration", "lower motor neuron degeneration",
-        "bulbar onset ALS", "spinal onset ALS",
-        "frontotemporal dementia", "respiratory failure",
-    ],
-}
-
 # ALS condition synonyms for ClinicalTrials.gov queries
 ALS_CONDITION_TERMS = [
     "Amyotrophic Lateral Sclerosis",

data/seeds/manual_seeds.json

@@ -0,0 +1,26 @@
+{
+  "genes": [
+    "SOD1", "TARDBP", "FUS", "C9orf72", "ATXN2",
+    "TBK1", "OPTN", "UBQLN2", "VCP", "NEK1",
+    "ANG", "SETX", "SIGMAR1", "CHCHD10", "MATR3"
+  ],
+  "proteins": [
+    "TDP-43", "FUS protein", "SOD1 protein", "Alsin",
+    "Optineurin", "p62", "Ubiquilin-2"
+  ],
+  "compounds": [
+    "riluzole", "edaravone", "tofersen", "AMX0035",
+    "mexiletine", "masitinib", "bosutinib"
+  ],
+  "mechanisms": [
+    "glutamate excitotoxicity", "oxidative stress",
+    "neuroinflammation", "protein aggregation",
+    "RNA metabolism dysfunction", "mitochondrial dysfunction",
+    "axonal transport defect", "autophagy impairment"
+  ],
+  "phenotypes": [
+    "upper motor neuron degeneration", "lower motor neuron degeneration",
+    "bulbar onset ALS", "spinal onset ALS",
+    "frontotemporal dementia", "respiratory failure"
+  ]
+}

extraction/extractor.py

@@ -19,7 +19,7 @@ from config import (
     EXTRACTION_PROGRESS_PATH,
     PAPERS_PATH,
 )
-from extraction.normalizer import CanonicalRegistry, normalize_entity
+from extraction.normalizer import CanonicalRegistry, guess_entity_type, normalize_entity
 from logging_config import get_logger
 from models import ALSPaper, ExtractedEntity, EntityRelationship, PaperExtractionResult
 from tools import EXTRACTION_TOOLS
@@ -254,14 +254,7 @@ def _parse_relationships(
     return rels
 
 
-def _guess_type(name: str) -> str:
-    """Best-effort entity type guess from name for relationship source/target."""
-    from extraction.normalizer import _GENE_ALIASES, _COMPOUND_ALIASES
-    if name.strip().upper() in _GENE_ALIASES or name.strip() in _GENE_ALIASES:
-        return "Gene"
-    if name.strip() in _COMPOUND_ALIASES:
-        return "Compound"
-    return "Protein"
+_guess_type = guess_entity_type
 
 
 def _load_papers(path: Path) -> list[ALSPaper]:

extraction/normalizer.py

@@ -104,6 +104,16 @@ _MECHANISM_ALIASES: dict[str, str] = {
 }
 
 
+def guess_entity_type(name: str) -> str:
+    """Best-effort entity type inference from name. Used when type metadata is unavailable."""
+    n = name.strip()
+    if n.upper() in _GENE_ALIASES or n in _GENE_ALIASES:
+        return "Gene"
+    if n in _COMPOUND_ALIASES:
+        return "Compound"
+    return "Protein"
+
+
 def normalize_entity(name: str, entity_type: str) -> str:
     """Return a canonical_id string in the form '<prefix>:<canonical_name>'."""
     canonical = _resolve_name(name.strip(), entity_type)

graph/builder.py

@@ -11,9 +11,10 @@ from pathlib import Path
 import networkx as nx
 
 from config import (
-    ALS_SEED_ENTITIES,
+    DERIVED_SEEDS_PATH,
     ENTITIES_PATH,
     KG_MIN_EDGE_CONFIDENCE,
+    MANUAL_SEEDS_PATH,
     TRIALS_PATH,
 )
 from extraction.normalizer import normalize_entity
@@ -59,9 +60,18 @@ def _add_seed_entities(G: nx.DiGraph) -> None:
         "mechanisms": "Mechanism",
         "phenotypes": "Phenotype",
     }
+
+    manual = json.loads(MANUAL_SEEDS_PATH.read_text())
+
+    derived: dict[str, list[str]] = {}
+    if DERIVED_SEEDS_PATH.exists():
+        derived = json.loads(DERIVED_SEEDS_PATH.read_text())
+
     for category, entity_type in type_map.items():
-        for name in ALS_SEED_ENTITIES.get(category, []):
+        seen: set[str] = set()
+        for name in manual.get(category, []):
             canonical_id = normalize_entity(name, entity_type)
+            seen.add(canonical_id)
             _upsert_node(G, canonical_id, {
                 "type": entity_type,
                 "display_name": name,
@@ -69,6 +79,19 @@ def _add_seed_entities(G: nx.DiGraph) -> None:
                 "evidence_pmids": [],
                 "is_seed": True,
             })
+        for name in derived.get(category, []):
+            canonical_id = normalize_entity(name, entity_type)
+            if canonical_id in seen:
+                continue
+            seen.add(canonical_id)
+            _upsert_node(G, canonical_id, {
+                "type": entity_type,
+                "display_name": name,
+                "paper_count": 0,
+                "evidence_pmids": [],
+                "is_seed": False,
+                "is_derived_seed": True,
+            })
 
 
 def _add_extracted_entities(G: nx.DiGraph, entities_path: Path) -> int:
@@ -186,7 +209,27 @@ def _add_trials(G: nx.DiGraph, trials_path: Path) -> int:
                         matched = True
                         break
                 if not matched:
-                    _logger.debug(f"Trial {nct_id}: no graph node for target {target_name!r}")
+                    # Novel entity from trial with no paper evidence — canonicalize as
+                    # compound (trial targets are drugs) but type stays Unknown since
+                    # we can't confirm mechanism/class without literature support.
+                    fallback_id = normalize_entity(target_name, "Compound")
+                    _upsert_node(G, fallback_id, {
+                        "type": "Unknown",
+                        "display_name": target_name,
+                        "paper_count": 0,
+                        "evidence_pmids": [],
+                        "confidence": 0.0,
+                        "is_seed": False,
+                        "is_trial_derived": True,
+                    })
+                    G.add_edge(node_id, fallback_id, **{
+                        "relation_type": "TESTED_IN",
+                        "relation_types": ["TESTED_IN"],
+                        "evidence_pmids": [],
+                        "confidence": 1.0,
+                        "evidence_text": "",
+                    })
+                    _logger.debug(f"Trial {nct_id}: created trial-derived node {fallback_id!r} (type=Unknown)")
 
             n_trials += 1
     return n_trials

scripts/derive_seeds.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""
+Derive dynamic seed entities from the ingested trial and paper corpus.
+
+Sources:
+  1. trials.jsonl — DRUG/BIOLOGICAL interventions + LLM-extracted targets
+  2. entities.jsonl — entities appearing in >= SEED_PROMOTION_THRESHOLD papers
+
+Writes data/seeds/derived_seeds.json (always overwritten, never hand-edited).
+Run after extract_entities.py and before build_graph.py.
+
+Usage:
+    uv run python scripts/derive_seeds.py
+    uv run python scripts/derive_seeds.py --threshold 3
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from rich.console import Console
+
+from config import (
+    DERIVED_SEEDS_PATH,
+    ENTITIES_PATH,
+    SEED_PROMOTION_THRESHOLD,
+    TRIALS_PATH,
+)
+from extraction.normalizer import normalize_entity, _GENE_ALIASES, _COMPOUND_ALIASES
+
+console = Console()
+
+_INTERVENTION_BLOCKLIST = {
+    "placebo", "standard_care", "standard_of_care", "exercise",
+    "physical_therapy", "physiotherapy", "occupational_therapy",
+    "riluzole",  # already in manual seeds
+    "edaravone",
+    "best_supportive_care", "nutritional_support", "sham",
+    "observation", "usual_care",
+}
+
+_EMPTY_SEEDS: dict[str, list[str]] = {
+    "genes": [], "proteins": [], "compounds": [], "mechanisms": [], "phenotypes": [],
+}
+
+_TYPE_TO_CATEGORY = {
+    "Gene": "genes",
+    "Protein": "proteins",
+    "Compound": "compounds",
+    "Mechanism": "mechanisms",
+    "Phenotype": "phenotypes",
+    "Pathway": "mechanisms",
+}
+
+
+def _derive_from_trials(trials_path: Path) -> dict[str, set[str]]:
+    """Return category → set of display names derived from active trials."""
+    result: dict[str, set[str]] = defaultdict(set)
+
+    if not trials_path.exists():
+        console.print(f"[yellow]trials.jsonl not found at {trials_path} — skipping trial seeds[/yellow]")
+        return result
+
+    n_trials = 0
+    with open(trials_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            trial = json.loads(line)
+            n_trials += 1
+
+            # Drug/biological interventions from ClinicalTrials.gov structured data
+            for iv in trial.get("interventions", []):
+                if iv.get("type", "").upper() not in ("DRUG", "BIOLOGICAL"):
+                    continue
+                name = iv.get("name", "").strip()
+                if not name:
+                    continue
+                canonical = normalize_entity(name, "Compound")
+                slug = canonical.split(":", 1)[-1]
+                if slug not in _INTERVENTION_BLOCKLIST:
+                    result["compounds"].add(name)
+
+            # LLM-extracted target entities
+            for target in trial.get("target_entities", []):
+                target = target.strip()
+                if not target:
+                    continue
+                # Infer type via alias tables
+                u = target.upper()
+                if u in _GENE_ALIASES or target in _GENE_ALIASES:
+                    result["genes"].add(target)
+                elif target in _COMPOUND_ALIASES:
+                    result["compounds"].add(target)
+                else:
+                    # Unknown — put in compounds (most unmatched trial targets are drugs)
+                    slug = normalize_entity(target, "Compound").split(":", 1)[-1]
+                    if slug not in _INTERVENTION_BLOCKLIST:
+                        result["compounds"].add(target)
+
+    console.print(f"  Scanned {n_trials} trials")
+    return result
+
+
+def _derive_from_entities(entities_path: Path, threshold: int) -> dict[str, set[str]]:
+    """Return category → set of display names for entities in >= threshold papers."""
+    if not entities_path.exists():
+        console.print(f"[yellow]entities.jsonl not found at {entities_path} — skipping paper seeds[/yellow]")
+        return defaultdict(set)
+
+    # canonical_id → {type, display_name, count}
+    counts: dict[str, dict] = {}
+
+    with open(entities_path, encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            raw = json.loads(line)
+            for ent in raw.get("entities", []):
+                if ent.get("confidence", 0.0) < 0.6:
+                    continue
+                cid = ent.get("canonical_id", "")
+                if not cid:
+                    continue
+                if cid not in counts:
+                    counts[cid] = {
+                        "type": ent.get("type", "Unknown"),
+                        "display_name": ent.get("name", cid.split(":", 1)[-1]),
+                        "count": 0,
+                    }
+                counts[cid]["count"] += 1
+
+    result: dict[str, set[str]] = defaultdict(set)
+    promoted = 0
+    for cid, data in counts.items():
+        if data["count"] >= threshold:
+            category = _TYPE_TO_CATEGORY.get(data["type"])
+            if category:
+                result[category].add(data["display_name"])
+                promoted += 1
+
+    console.print(f"  {len(counts)} unique entities; {promoted} promoted above threshold={threshold}")
+    return result
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Derive seed entities from corpus")
+    parser.add_argument("--trials", type=Path, default=TRIALS_PATH)
+    parser.add_argument("--entities", type=Path, default=ENTITIES_PATH)
+    parser.add_argument("--threshold", type=int, default=SEED_PROMOTION_THRESHOLD)
+    parser.add_argument("--output", type=Path, default=DERIVED_SEEDS_PATH)
+    args = parser.parse_args()
+
+    console.print("[cyan]Deriving seeds from trials...[/cyan]")
+    trial_seeds = _derive_from_trials(args.trials)
+
+    console.print("[cyan]Deriving seeds from paper entities...[/cyan]")
+    paper_seeds = _derive_from_entities(args.entities, args.threshold)
+
+    merged: dict[str, list[str]] = {}
+    for category in _EMPTY_SEEDS:
+        combined = trial_seeds.get(category, set()) | paper_seeds.get(category, set())
+        merged[category] = sorted(combined)
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(merged, indent=2, sort_keys=True))
+
+    total = sum(len(v) for v in merged.values())
+    console.print(f"\n[bold green]Done![/bold green] Written to {args.output}")
+    console.print(f"  Total derived seeds: {total}")
+    for cat, names in merged.items():
+        if names:
+            console.print(f"  {cat}: {len(names)} ({', '.join(names[:5])}{'...' if len(names) > 5 else ''})")
+
+
+if __name__ == "__main__":
+    main()

scripts/ingest_papers.py

@@ -28,20 +28,27 @@ from rich.progress import track
 import ingestion.pmc as pmc
 import ingestion.pubmed as pubmed
 import ingestion.semantic_scholar as ss
-from config import PAPERS_PATH, PUBMED_DEFAULT_MAX, PUBMED_DEFAULT_QUERY
+from config import PAPERS_PATH, PUBMED_DEFAULT_MAX, PUBMED_DEFAULT_QUERY, PUBMED_REFRESH_QUERY_TEMPLATE
 
 console = Console()
 
 
 def main() -> None:
     parser = argparse.ArgumentParser(description="Ingest ALS papers")
-    parser.add_argument("--query", default=PUBMED_DEFAULT_QUERY, help="PubMed query string")
+    parser.add_argument("--query", default=None, help="PubMed query string (overrides --since)")
+    parser.add_argument("--since", metavar="YYYY-MM-DD", help="Fetch papers published on or after this date (incremental append mode)")
     parser.add_argument("--max", type=int, default=PUBMED_DEFAULT_MAX, dest="max_results", help="Max papers to fetch")
     parser.add_argument("--pmid-file", help="Path to file with one PMID per line")
     parser.add_argument("--skip-fulltext", action="store_true", help="Skip PMC full text fetch")
     parser.add_argument("--skip-citations", action="store_true", help="Skip Semantic Scholar citation counts")
     args = parser.parse_args()
 
+    if args.query is None:
+        if args.since:
+            args.query = PUBMED_REFRESH_QUERY_TEMPLATE.format(since_date=args.since)
+        else:
+            args.query = PUBMED_DEFAULT_QUERY
+
     PAPERS_PATH.parent.mkdir(parents=True, exist_ok=True)
 
     # Step 1: Obtain PMIDs
@@ -58,6 +65,21 @@ def main() -> None:
         console.print("[red]No PMIDs found — check your query or PMID file.[/red]")
         sys.exit(1)
 
+    # In append mode, filter out PMIDs already in the existing corpus
+    existing_pmids: set[str] = set()
+    if args.since and PAPERS_PATH.exists():
+        with open(PAPERS_PATH, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    existing_pmids.add(json.loads(line).get("pmid", ""))
+        original_count = len(pmids)
+        pmids = [p for p in pmids if p not in existing_pmids]
+        console.print(f"[cyan]Incremental mode: {original_count} fetched, {len(existing_pmids)} already known, {len(pmids)} new[/cyan]")
+        if not pmids:
+            console.print("[green]No new papers found — corpus is up to date.[/green]")
+            return
+
     # Step 2: Fetch paper records from PubMed
     console.print("[cyan]Fetching Medline records...[/cyan]")
     papers = pubmed.fetch_by_pmids(pmids)
@@ -91,8 +113,9 @@ def main() -> None:
             paper.citation_count = citation_map.get(paper.pmid, 0)
         console.print(f"[green]Got citation counts for {len(citation_map)}/{len(papers)} papers[/green]")
 
-    # Step 5: Write to JSONL
-    with open(PAPERS_PATH, "w", encoding="utf-8") as f:
+    # Step 5: Write to JSONL (append in incremental mode, overwrite otherwise)
+    write_mode = "a" if args.since else "w"
+    with open(PAPERS_PATH, write_mode, encoding="utf-8") as f:
         for paper in papers:
             f.write(json.dumps(paper.to_dict()) + "\n")
 

scripts/ingest_trials.py

@@ -44,6 +44,7 @@ def main() -> None:
             f"ACTIVE_NOT_RECRUITING). Choices: {_ALL_STATUSES}"
         ),
     )
+    parser.add_argument("--upsert", action="store_true", help="Merge fetched trials into existing trials.jsonl by nct_id")
     args = parser.parse_args()
 
     TRIALS_PATH.parent.mkdir(parents=True, exist_ok=True)
@@ -54,6 +55,20 @@ def main() -> None:
     trials = fetch_als_trials(status=args.status, client=client)
     console.print(f"[green]Fetched {len(trials)} trials[/green]")
 
+    if args.upsert and TRIALS_PATH.exists():
+        existing: dict[str, dict] = {}
+        with open(TRIALS_PATH, encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line:
+                    t = json.loads(line)
+                    existing[t["nct_id"]] = t
+        before = len(existing)
+        for trial in trials:
+            existing[trial["nct_id"]] = trial
+        trials = list(existing.values())
+        console.print(f"[cyan]Upsert: {before} existing + {len(trials) - before} new/updated → {len(trials)} total[/cyan]")
+
     with open(TRIALS_PATH, "w", encoding="utf-8") as f:
         for trial in trials:
             f.write(json.dumps(trial) + "\n")

scripts/refresh.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+Incremental refresh orchestrator. Runs the full pipeline in order, using
+--since for papers and --upsert for trials to avoid full re-ingestion.
+
+State is tracked in data/.refresh_state.json. On first run (no state file),
+defaults to fetching papers since 2024-12-31 (end of the initial corpus window).
+
+Usage:
+    uv run python scripts/refresh.py                    # incremental from last state
+    uv run python scripts/refresh.py --since 2025-01-01
+    uv run python scripts/refresh.py --dry-run
+    uv run python scripts/refresh.py --skip-papers      # trials + graph only
+    uv run python scripts/refresh.py --full-rebuild     # ignore state, full re-ingest
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from datetime import date
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from rich.console import Console
+
+from config import REFRESH_STATE_PATH
+
+console = Console()
+
+_SCRIPTS_DIR = Path(__file__).parent
+_DEFAULT_SINCE = "2024-12-31"
+
+
+def _load_state() -> dict:
+    if REFRESH_STATE_PATH.exists():
+        return json.loads(REFRESH_STATE_PATH.read_text())
+    return {}
+
+
+def _save_state(state: dict) -> None:
+    REFRESH_STATE_PATH.parent.mkdir(parents=True, exist_ok=True)
+    REFRESH_STATE_PATH.write_text(json.dumps(state, indent=2))
+
+
+def _run(script: str, *args: str, dry_run: bool = False) -> None:
+    cmd = [sys.executable, str(_SCRIPTS_DIR / script), *args]
+    console.print(f"[cyan]{'(dry-run) ' if dry_run else ''}Running:[/cyan] {' '.join(cmd)}")
+    if dry_run:
+        return
+    result = subprocess.run(cmd, check=False)
+    if result.returncode != 0:
+        console.print(f"[red]Script {script} exited with code {result.returncode}[/red]")
+        sys.exit(result.returncode)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Incremental pipeline refresh")
+    parser.add_argument("--since", metavar="YYYY-MM-DD", help="Override since-date for paper ingestion")
+    parser.add_argument("--dry-run", action="store_true", help="Print what would run without executing")
+    parser.add_argument("--skip-papers", action="store_true", help="Skip paper ingestion (trials + graph only)")
+    parser.add_argument("--full-rebuild", action="store_true", help="Full re-ingest, ignoring state")
+    args = parser.parse_args()
+
+    state = _load_state()
+    today = str(date.today())
+
+    if args.full_rebuild:
+        console.print("[yellow]Full rebuild requested — ignoring refresh state[/yellow]")
+        since_date = None
+    elif args.since:
+        since_date = args.since
+    else:
+        since_date = state.get("last_papers_ingest", _DEFAULT_SINCE)
+
+    console.print(f"[bold]Candle-fire incremental refresh[/bold] — {today}")
+    if not args.full_rebuild and not args.skip_papers:
+        console.print(f"  Fetching papers since: {since_date}")
+
+    # 1. Papers
+    if not args.skip_papers:
+        if args.full_rebuild or since_date is None:
+            _run("ingest_papers.py", "--skip-fulltext", "--skip-citations", dry_run=args.dry_run)
+        else:
+            _run("ingest_papers.py", "--since", since_date, "--skip-fulltext", "--skip-citations", dry_run=args.dry_run)
+
+    # 2. Trials
+    _run("ingest_trials.py", "--upsert", dry_run=args.dry_run)
+
+    # 3. Entity extraction (auto-resumes via .progress.json)
+    _run("extract_entities.py", dry_run=args.dry_run)
+
+    # 4. Derive seeds
+    _run("derive_seeds.py", dry_run=args.dry_run)
+
+    # 5. Build graph
+    _run("build_graph.py", dry_run=args.dry_run)
+
+    # 6. Build index (idempotent — skips existing chunks)
+    _run("build_index.py", dry_run=args.dry_run)
+
+    if not args.dry_run:
+        new_state = {**state, "last_papers_ingest": today, "last_trials_ingest": today}
+        _save_state(new_state)
+        console.print(f"\n[bold green]Refresh complete.[/bold green] State written to {REFRESH_STATE_PATH}")
+    else:
+        console.print("\n[yellow]Dry run complete — no changes made.[/yellow]")
+
+
+if __name__ == "__main__":
+    main()