scripts/build_open_problems.py

"""
Open-Problems scorer — find gaps in the concept graph that no paper has
addressed yet, and rank them by how much novelty + value they'd unlock.

Three kinds of gap:

  1. missing_bridge: two high-paper-degree concepts that *should* connect
     (both are foundational, both appear together as activations in
     some paper's foundation set, yet have zero edges between them in
     the corpus). A new paper could earn high novelty bits by linking
     them.

  2. underexplored: a wiki concept that exists in the vocabulary and
     is referenced as a foundation by multiple other papers, but only
     1-2 papers in the corpus actively activate it as a primary
     concept. A focused paper on it would close a key gap.

  3. stale_cluster: a Louvain-style cluster (computed by greedy
     co-occurrence seeding) whose newest activating paper is older
     than the cluster's median paper year. The community has gone
     quiet — open territory for a fresh contribution.

For each problem we attach:
  - closest_papers: top-3 arXivis papers most likely to be relevant
    (concept overlap w/ the problem's involved nodes).
  - suggested_foundations: high-paper-degree concepts the new work
    could build on.

Output: design/open_problems.js (window.OPEN_PROBLEMS).

Run:
  python3 scripts/build_open_problems.py
  python3 scripts/build_open_problems.py --max 60
"""

from __future__ import annotations

import argparse
import json
import sys
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path

PROJECT_ROOT = Path(__file__).parent.parent
ARXIVIS_JS   = PROJECT_ROOT / "design" / "arxivis_data.js"
OUT_PATH     = PROJECT_ROOT / "design" / "open_problems.js"


def load_arxivis() -> dict:
    if not ARXIVIS_JS.exists():
        sys.exit(f"missing {ARXIVIS_JS} — run scripts/build_arxivis_data.py first")
    t = ARXIVIS_JS.read_text()
    return json.loads(t[t.find("{"): t.rfind("}") + 1])


GENERIC_STOP = {
    "filter", "reduce", "partition", "match", "map", "graph", "tree",
    "set", "node", "edge", "function", "vector", "matrix", "value",
    "norm", "rate", "loss", "rule", "step", "task", "limit", "row",
    "column", "operator", "label",
    "convert", "split", "merge", "build", "init", "scan", "test",
}


def _is_real_concept(name: str) -> bool:
    """Skip noisy single-verb tokens that aren't really concepts."""
    if not name: return False
    if len(name) < 5: return False
    if name.lower().strip() in GENERIC_STOP: return False
    return True


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--max", type=int, default=50, help="problems to keep")
    args = ap.parse_args()

    data = load_arxivis()
    papers   = data["papers"]
    universe = data.get("concept_universe", [])
    edges    = data.get("concept_edges", [])  # [a, b, weight]

    # Concept lookup tables
    concept_by_id = {c["id"]: c for c in universe}
    # Paper-degree (count of papers activating this concept) = universe.degree
    # Edge weight (count of papers in which both co-activate) = edges[i][2]
    edge_weight: dict[tuple[str, str], int] = {}
    for e in edges:
        a, b, w = e[0], e[1], (e[2] if len(e) > 2 else 1)
        edge_weight[(a, b)] = w
        edge_weight[(b, a)] = w

    # Per-paper concept set (slug form) for relevance lookup
    paper_concept_set: dict[str, set[str]] = {}
    paper_year: dict[str, int | None] = {}
    for p in papers:
        ids = {c["id"].replace("wiki:", "") for c in p.get("concepts", [])}
        paper_concept_set[p["id"]] = ids
        paper_year[p["id"]] = p.get("year")

    def closest_papers(involved_ids: set[str], k: int = 3) -> list[dict]:
        scored = []
        for p in papers:
            inter = paper_concept_set[p["id"]] & involved_ids
            if not inter:
                continue
            scored.append({
                "id":      p["id"],
                "title":   p["title"],
                "overlap": len(inter),
                "novelty": p.get("novelty_bits", 0),
            })
        scored.sort(key=lambda x: (-x["overlap"], -x["novelty"]))
        return scored[:k]

    def name(slug: str) -> str:
        c = concept_by_id.get(slug)
        if c:
            return c["name"]
        return slug.replace("_", " ")

    problems = []

    # ─── 1. MISSING BRIDGES ─────────────────────────────────────────────
    # Concepts with paper-degree ≥ 3 that have no direct edge despite
    # appearing together as activations in some paper's pair_edges.
    # We approximate this with: cluster of high-degree concepts whose
    # mutual edge_weight is 0.
    high_deg = [c for c in universe
                if c.get("degree", 0) >= 3 and _is_real_concept(c.get("name", ""))]
    # Sort by degree descending, cap to keep N^2 manageable
    high_deg.sort(key=lambda c: -c["degree"])
    high_deg = high_deg[:60]

    # Cluster co-membership = same cluster id from build_arxivis_data
    seen_pairs: set[tuple[str, str]] = set()
    for i, a in enumerate(high_deg):
        for b in high_deg[i + 1:]:
            key = tuple(sorted([a["id"], b["id"]]))
            if key in seen_pairs:
                continue
            seen_pairs.add(key)
            w = edge_weight.get(key, 0)
            if w > 0:
                continue
            # Higher score for pairs across different clusters
            cross_cluster = a.get("cluster") != b.get("cluster")
            score = (a["degree"] + b["degree"]) * (1.6 if cross_cluster else 1.0)
            problems.append({
                "kind":         "missing_bridge",
                "title":        f"No paper has bridged {name(a['id'])} and {name(b['id'])}",
                "concepts":     [a["id"], b["id"]],
                "score":        round(score, 2),
                "why":          (
                    f"{name(a['id'])} appears in {a['degree']} papers, "
                    f"{name(b['id'])} in {b['degree']}, but the corpus "
                    f"contains zero papers that activate both."
                    + (" Bridges two distinct clusters."
                       if cross_cluster else "")
                ),
                "closest_papers": closest_papers({a["id"], b["id"]}),
                "suggested_foundations": [
                    {"id": a["id"], "name": name(a["id"]), "papers": a["degree"]},
                    {"id": b["id"], "name": name(b["id"]), "papers": b["degree"]},
                ],
            })

    # ─── 2. UNDEREXPLORED ANCHORS ───────────────────────────────────────
    # Concepts mentioned as `foundations` by ≥ 3 papers, but themselves
    # activated as a primary concept in ≤ 2 papers. Foundation pull
    # without active research = open territory.
    foundation_pull: Counter = Counter()
    for p in papers:
        for f in p.get("foundations", []) or []:
            # foundations were stored w/ raw id (e.g. wiki:slug); strip
            slug = f["id"].replace("wiki:", "") if "id" in f else None
            if slug:
                foundation_pull[slug] += 1
    for slug, pull in foundation_pull.items():
        c = concept_by_id.get(slug)
        if not c:
            continue
        if not _is_real_concept(c.get("name", "")):
            continue
        deg = c.get("degree", 0)
        if pull >= 2 and deg <= 2:
            score = pull * 4 + (3 - deg)
            problems.append({
                "kind":         "underexplored",
                "title":        f"{name(slug)} is foundational but barely studied",
                "concepts":     [slug],
                "score":        round(score, 2),
                "why":          (
                    f"{pull} papers in the corpus build ON {name(slug)} as a "
                    f"foundation, yet only {deg} paper{'s' if deg != 1 else ''} "
                    f"focuses on it as a primary subject."
                ),
                "closest_papers": closest_papers({slug}),
                "suggested_foundations": [
                    {"id": slug, "name": name(slug), "papers": deg},
                ],
            })

    # ─── 3. STALE CLUSTERS ──────────────────────────────────────────────
    # For each cluster id, find median year of activating papers + newest.
    # Stale = newest is older than median (community has slowed).
    cluster_papers: dict[int, list[int]] = defaultdict(list)
    cluster_concepts: dict[int, list[str]] = defaultdict(list)
    for c in universe:
        cluster_concepts[c.get("cluster", 0)].append(c["id"])
    for p in papers:
        y = p.get("year")
        if not y:
            continue
        clusters_touched = set()
        for c in p.get("concepts", []):
            slug = c["id"].replace("wiki:", "")
            cid = concept_by_id.get(slug, {}).get("cluster")
            if cid is not None:
                clusters_touched.add(cid)
        for cid in clusters_touched:
            cluster_papers[cid].append(y)

    for cid, years in cluster_papers.items():
        if len(years) < 4:
            continue
        years_sorted = sorted(years)
        median = years_sorted[len(years_sorted) // 2]
        newest = years_sorted[-1]
        # If the newest paper isn't notably newer than the median, cluster's stale
        if newest - median <= 0:
            seed_concepts = [name(s) for s in cluster_concepts[cid][:3]]
            score = len(years) * 0.5 + (median - 2000)
            problems.append({
                "kind":         "stale_cluster",
                "title":        f"Cluster around {', '.join(seed_concepts)} has gone quiet",
                "concepts":     cluster_concepts[cid][:5],
                "score":        round(score, 2),
                "why":          (
                    f"{len(years)} papers in this cluster, median year "
                    f"{median}, newest paper {newest}. No recent activity — "
                    "open territory for a fresh contribution."
                ),
                "closest_papers": closest_papers(set(cluster_concepts[cid][:8])),
                "suggested_foundations": [
                    {"id": s, "name": name(s), "papers": concept_by_id.get(s, {}).get("degree", 1)}
                    for s in cluster_concepts[cid][:3]
                ],
            })

    # Bucket-balance: ensure variety across kinds (avoid 60 missing_bridge dominating)
    by_kind: dict[str, list[dict]] = defaultdict(list)
    for p in problems:
        by_kind[p["kind"]].append(p)
    for k in by_kind:
        by_kind[k].sort(key=lambda p: -p["score"])
    balanced: list[dict] = []
    quota = max(args.max // 3, 5)
    for k in ("missing_bridge", "underexplored", "stale_cluster"):
        balanced.extend(by_kind.get(k, [])[:quota])
    # Top up w/ remaining missing_bridges if room
    while len(balanced) < args.max:
        rest = [p for k in by_kind for p in by_kind[k] if p not in balanced]
        if not rest:
            break
        rest.sort(key=lambda p: -p["score"])
        balanced.append(rest[0])
    balanced.sort(key=lambda p: -p["score"])
    problems = balanced[: args.max]

    out = {
        "problems":     problems,
        "generated_at": datetime.now(timezone.utc).isoformat(),
        "stats": {
            "corpus_papers":      len(papers),
            "universe_concepts":  len(universe),
            "missing_bridges":    sum(1 for p in problems if p["kind"] == "missing_bridge"),
            "underexplored":      sum(1 for p in problems if p["kind"] == "underexplored"),
            "stale_clusters":     sum(1 for p in problems if p["kind"] == "stale_cluster"),
        },
    }
    js = "window.OPEN_PROBLEMS = " + json.dumps(out, indent=2, ensure_ascii=False) + ";\n"
    OUT_PATH.write_text(js)
    print(f"→ wrote {OUT_PATH} ({OUT_PATH.stat().st_size // 1024} KB)")
    print(f"  {len(problems)} problems: {out['stats']}")
    for p in problems[:5]:
        print(f"  [{p['kind']:14s} {p['score']:6.1f}] {p['title']}")


if __name__ == "__main__":
    main()