Bake grown graph into the Space: 63 nodes (35 curated + 28 discovered)

forge/grow.py

@@ -0,0 +1,82 @@
+"""Node discovery — grow the graph's component ("tool") count from scraped pages.
+
+The edge extractor only links tools already in the graph; this pass finds NEW
+tools named in scraped content and adds them as nodes. Gated: a candidate must
+appear in >= min_sources distinct sources before it's added, so we don't fill
+the graph with one-off junk. New nodes are tagged 'discovered' (review-pending).
+
+Replays saved campaign content — no new Bright Data credits.
+
+Run:  python -m forge.grow
+"""
+from __future__ import annotations
+
+import glob
+import json
+from collections import Counter
+
+from . import collect, db, scraper
+
+VALID_TYPES = {"optimizer", "scheduler", "technique", "quantization",
+               "architecture", "inference", "framework"}
+
+
+def run(min_sources=2):
+    conn = db.connect()
+    ai = scraper.build_alias_index(conn)
+    known = {scraper._norm(a) for a, _ in ai}
+    before = conn.execute("SELECT COUNT(*) FROM nodes").fetchone()[0]
+
+    files = sorted(glob.glob(str(collect.RAW_DIR / "forge_campaign_*.json")))
+    files += sorted(glob.glob(str(collect.RAW_DIR / "forge_collect_*.json")))
+    cand, pages = {}, 0
+    for f in files:
+        bundle = json.load(open(f))
+        for plist in bundle.get("passes", {}).values():
+            if not isinstance(plist, list):
+                continue
+            for item in plist:
+                content = item.get("content") or ""
+                url = item.get("link") or item.get("url") or ""
+                if len(content) < 200:
+                    continue
+                pages += 1
+                for comp in scraper.extract_components(content):
+                    norm = scraper._norm(comp.get("name", ""))
+                    ty = comp.get("type", "")
+                    if not norm or norm in known or ty not in VALID_TYPES:
+                        continue
+                    c = cand.setdefault(norm, {"display": comp["name"].strip(), "types": [], "sources": set()})
+                    c["types"].append(ty)
+                    if url:
+                        c["sources"].add(url)
+
+    added = []
+    for norm, info in cand.items():
+        if len(info["sources"]) >= min_sources:
+            ty = Counter(info["types"]).most_common(1)[0][0]
+            try:
+                db.add_node(conn, type=ty, name=info["display"], canonical=norm,
+                            aliases=[], description="Discovered via Bright Data scraping (review-pending).",
+                            tags=["discovered"])
+                known.add(norm)
+                added.append((info["display"], ty, len(info["sources"])))
+            except Exception:  # noqa: BLE001 - duplicate canonical race
+                pass
+    conn.commit()
+    after = conn.execute("SELECT COUNT(*) FROM nodes").fetchone()[0]
+    return conn, dict(pages=pages, candidates=len(cand), added=added, before=before, after=after)
+
+
+def main():
+    conn, r = run()
+    print(f"pages scanned     : {r['pages']}")
+    print(f"new candidates    : {r['candidates']}")
+    print(f"nodes: {r['before']} -> {r['after']}  (+{r['after'] - r['before']})")
+    print("new tools added (>=2 sources):")
+    for name, ty, n in sorted(r["added"], key=lambda x: -x[2]):
+        print(f"   + {name}  [{ty}]  ({n} sources)")
+
+
+if __name__ == "__main__":
+    main()

forge/scraper.py

@@ -226,6 +226,36 @@ def _llm_up() -> bool:
         return False
 
 
+_NODE_PROMPT = """List the ML TRAINING components/tools/methods NAMED in this text.
+Categories: optimizer, scheduler, technique, quantization, architecture, inference, framework.
+Return JSON: {"components":[{"name":"<specific named tool>","type":"<category>"}]}.
+Only specific named tools/methods (e.g. GaLore, DoRA, FlashAttention, Sophia, Adafactor,
+Megatron, Axolotl, ReLoRA, Shampoo). NO generic words, NO prose. If none: {"components":[]}."""
+
+
+def extract_components(markdown: str) -> list[dict]:
+    """Local-LLM node discovery — names + categories of training tools mentioned."""
+    if not _llm_up():
+        return []
+    payload = {
+        "messages": [{"role": "system", "content": _NODE_PROMPT},
+                     {"role": "user", "content": _clean_markdown(markdown)[:12000]}],
+        "temperature": 0, "max_tokens": 1200, "response_format": {"type": "json_object"},
+    }
+    req = urllib.request.Request(
+        LLM_URL.rstrip("/") + "/v1/chat/completions", data=json.dumps(payload).encode(),
+        headers={"Content-Type": "application/json", "Authorization": "Bearer no-key"})
+    try:
+        with urllib.request.urlopen(req, timeout=120) as r:
+            resp = json.loads(r.read())
+        text = resp["choices"][0]["message"]["content"]
+        m = re.search(r"\{[\s\S]*\}", text)
+        obj = json.loads(m.group(0)) if m else {}
+        return [c for c in obj.get("components", []) if c.get("name") and c.get("type")]
+    except Exception:  # noqa: BLE001
+        return []
+
+
 def _extract_local(markdown: str, alias_index) -> list[dict]:
     """Extract via a local llama.cpp OpenAI-compatible server. No API key."""
     vocab = sorted({c for _, c in alias_index})