"""
Build ResearchHierarchy-style JSONL spans and simple ontology metadata for WOS / arXiv papers.

This is a reference implementation of the dataset construction described for the
ResearchHierarchy benchmark in the TOLBERT paper. It expects you to provide a local
metadata file that assigns each paper to a 3-level research taxonomy and (optionally)
its backing PDF path.

The script is intentionally lightweight and mirrors the structure of
`build_codehierarchy_spans.py`:

Input
=====
- --metadata_file:
    A CSV or JSONL file with, at minimum, the columns:
      doc_id, field, subfield, discipline, text

    Optionally it may also include:
      pdf_path: absolute or repo-relative path to the PDF for this paper
      source:  short tag like "wos" or "arxiv" identifying the corpus

    You can override these column names with CLI flags.

Outputs
=======
- --spans_out:
    JSONL file with one record per paper of the form:
      {
        "span_id": "WOS-12345",
        "text": "... title + abstract or full text ...",
        "source_id": "/papers/pdfs/2019/WOS-12345.pdf",
        "node_path": [root_id, field_id, subfield_id, discipline_id],
        "meta": {
          "field": "...",
          "subfield": "...",
          "discipline": "...",
          "doc_id": "...",
          "pdf_path": "...",        # if available
          "source": "wos" | "arxiv" # if available
        }
      }

- --nodes_out (optional):
    JSONL file describing ontology nodes with fields:
      node_id, level, type, parent_id, name, attributes

- --level_sizes_out (optional):
    Small JSON file with:
      {"level_sizes": [num_level0, num_level1, num_level2, num_level3]}
    which you can copy into your training config as `level_sizes`.

The intended use is:
  1) Prepare a single metadata file that combines your WOS and arXiv papers,
     mapped into a 3-level taxonomy (field → subfield → discipline).
  2) Point this script at it to obtain spans and ontology metadata compatible
     with the rest of the TOLBERT pipeline.
"""

from __future__ import annotations

import argparse
import csv
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, List, Optional, Tuple

@dataclass
class PaperMeta:
    doc_id: str
    field: str
    subfield: str
    discipline: str
    text: str
    pdf_path: Optional[str] = None
    source: Optional[str] = None

def load_metadata(
    path: Path,
    *,
    id_col: str,
    field_col: str,
    subfield_col: str,
    discipline_col: str,
    text_col: str,
    pdf_path_col: Optional[str],
    source_col: Optional[str],
) -> List[PaperMeta]:
    """
    Load WOS / arXiv-style paper metadata from CSV or JSON(L).

    The loader is column-name agnostic: you can override the column names via CLI.
    """
    metas: List[PaperMeta] = []

    def _row_to_meta(row: Dict[str, object]) -> Optional[PaperMeta]:
        try:
            doc_id = str(row[id_col])
            field = str(row[field_col])
            subfield = str(row[subfield_col])
            discipline = str(row[discipline_col])
            text = str(row[text_col])
        except KeyError as exc:
            raise KeyError(f"Missing required column {exc!s} in metadata record: {row}") from exc

        if not text.strip():
            # Skip empty-text entries; they are not useful training instances.
            return None

        pdf_path_val: Optional[str] = None
        if pdf_path_col and pdf_path_col in row and row[pdf_path_col] is not None:
            pdf_path_val = str(row[pdf_path_col])

        source_val: Optional[str] = None
        if source_col and source_col in row and row[source_col] is not None:
            source_val = str(row[source_col])

        return PaperMeta(
            doc_id=doc_id,
            field=field,
            subfield=subfield,
            discipline=discipline,
            text=text,
            pdf_path=pdf_path_val,
            source=source_val,
        )

    suffix = path.suffix.lower()
    if suffix in {".jsonl", ".json"}:
        with path.open("r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                obj = json.loads(line)
                # Accept either a flat object or nested under "data"
                if isinstance(obj, dict) and "data" in obj and isinstance(obj["data"], dict):
                    obj = obj["data"]
                if not isinstance(obj, dict):
                    continue
                meta = _row_to_meta(obj)
                if meta is not None:
                    metas.append(meta)
        return metas

    # Default: CSV with header.
    with path.open("r", encoding="utf-8") as f:
        reader = csv.DictReader(f)
        for row in reader:
            meta = _row_to_meta(row)
            if meta is not None:
                metas.append(meta)
    return metas

def build_ontology(
    metas: List[PaperMeta],
) -> Tuple[Dict[str, int], Dict[str, int], Dict[str, int]]:
    """
    Build simple integer ID mappings:
      - field -> field_node_id
      - (field, subfield) -> subfield_node_id
      - (field, subfield, discipline) -> discipline_node_id

    Level conventions:
      level 0: root (id 0)
      level 1: field nodes
      level 2: subfield nodes
      level 3: discipline nodes
    """
    next_id = 0
    root_id = next_id
    next_id += 1

    field_ids: Dict[str, int] = {}
    subfield_ids: Dict[str, int] = {}
    discipline_ids: Dict[str, int] = {}

    for m in metas:
        if m.field not in field_ids:
            field_ids[m.field] = next_id
            next_id += 1

        sub_key = f"{m.field}::{m.subfield}"
        if sub_key not in subfield_ids:
            subfield_ids[sub_key] = next_id
            next_id += 1

        disc_key = f"{m.field}::{m.subfield}::{m.discipline}"
        if disc_key not in discipline_ids:
            discipline_ids[disc_key] = next_id
            next_id += 1

    # Sanity check: at least one node at each level if any metas were provided.
    if metas and (not field_ids or not subfield_ids or not discipline_ids):
        raise RuntimeError("Failed to build a non-empty 3-level ontology from metadata.")

    # root_id is currently unused in the returned dicts but kept for documentation.
    _ = root_id
    return field_ids, subfield_ids, discipline_ids

def write_nodes_jsonl(
    out_path: Path,
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
) -> None:
    """
    Emit a minimal nodes.jsonl compatible with docs/tree_of_life.md.
    """
    with out_path.open("w", encoding="utf-8") as f:
        # Root (level 0)
        f.write(
            json.dumps(
                {
                    "node_id": 0,
                    "level": 0,
                    "type": "root",
                    "parent_id": None,
                    "name": "Root",
                    "attributes": {},
                }
            )
            + "\n"
        )

        # Fields (level 1)
        for field_name, nid in field_ids.items():
            f.write(
                json.dumps(
                    {
                        "node_id": nid,
                        "level": 1,
                        "type": "field",
                        "parent_id": 0,
                        "name": field_name,
                        "attributes": {},
                    }
                )
                + "\n"
            )

        # Subfields (level 2)
        for key, nid in subfield_ids.items():
            field_name, subfield_name = key.split("::", 1)
            parent_id = field_ids[field_name]
            f.write(
                json.dumps(
                    {
                        "node_id": nid,
                        "level": 2,
                        "type": "subfield",
                        "parent_id": parent_id,
                        "name": subfield_name,
                        "attributes": {"field": field_name},
                    }
                )
                + "\n"
            )

        # Disciplines (level 3)
        for key, nid in discipline_ids.items():
            field_name, subfield_name, discipline_name = key.split("::", 2)
            sub_key = f"{field_name}::{subfield_name}"
            parent_id = subfield_ids[sub_key]
            f.write(
                json.dumps(
                    {
                        "node_id": nid,
                        "level": 3,
                        "type": "discipline",
                        "parent_id": parent_id,
                        "name": discipline_name,
                        "attributes": {
                            "field": field_name,
                            "subfield": subfield_name,
                        },
                    }
                )
                + "\n"
            )

def build_span_records(
    metas: List[PaperMeta],
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
) -> List[Dict[str, object]]:
    """
    Iterate over papers and generate one span per paper.
    """
    root_id = 0
    spans: List[Dict[str, object]] = []

    for m in metas:
        field_id = field_ids[m.field]
        sub_key = f"{m.field}::{m.subfield}"
        sub_id = subfield_ids[sub_key]
        disc_key = f"{m.field}::{m.subfield}::{m.discipline}"
        disc_id = discipline_ids[disc_key]

        node_path = [root_id, field_id, sub_id, disc_id]

        span_id = m.doc_id
        source_id = m.pdf_path if m.pdf_path is not None else m.doc_id

        meta: Dict[str, object] = {
            "doc_id": m.doc_id,
            "field": m.field,
            "subfield": m.subfield,
            "discipline": m.discipline,
        }
        if m.pdf_path is not None:
            meta["pdf_path"] = m.pdf_path
        if m.source is not None:
            meta["source"] = m.source

        spans.append(
            {
                "span_id": span_id,
                "text": m.text,
                "source_id": source_id,
                "node_path": node_path,
                "meta": meta,
            }
        )

    return spans

def write_spans_jsonl(spans: List[Dict[str, object]], out_path: Path) -> None:
    with out_path.open("w", encoding="utf-8") as f:
        for rec in spans:
            f.write(json.dumps(rec) + "\n")

def write_level_sizes(
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
    out_path: Path,
) -> None:
    # We always have exactly one root.
    level_sizes = [
        1,  # root
        len(field_ids),
        len(subfield_ids),
        len(discipline_ids),
    ]
    out = {"level_sizes": level_sizes}
    with out_path.open("w", encoding="utf-8") as f:
        json.dump(out, f, indent=2)

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(
        description=(
            "Build ResearchHierarchy spans JSONL and ontology metadata from WOS / arXiv metadata."
        )
    )
    ap.add_argument(
        "--metadata_file",
        type=str,
        required=True,
        help="CSV or JSONL file with at least: doc_id, field, subfield, discipline, text.",
    )
    ap.add_argument(
        "--spans_out",
        type=str,
        required=True,
        help="Output path for spans JSONL file.",
    )
    ap.add_argument(
        "--nodes_out",
        type=str,
        default="",
        help="Optional output path for nodes JSONL (ontology nodes).",
    )
    ap.add_argument(
        "--level_sizes_out",
        type=str,
        default="",
        help="Optional output path for level_sizes JSON file.",
    )

    # Column name overrides for flexibility.
    ap.add_argument("--id_col", type=str, default="doc_id", help="Column name for document ID.")
    ap.add_argument("--field_col", type=str, default="field", help="Column name for level-1 field.")
    ap.add_argument(
        "--subfield_col",
        type=str,
        default="subfield",
        help="Column name for level-2 subfield.",
    )
    ap.add_argument(
        "--discipline_col",
        type=str,
        default="discipline",
        help="Column name for level-3 discipline.",
    )
    ap.add_argument(
        "--text_col",
        type=str,
        default="text",
        help="Column name containing the paper text (e.g., title + abstract).",
    )
    ap.add_argument(
        "--pdf_path_col",
        type=str,
        default="pdf_path",
        help="Optional column name for the PDF path; if missing, source_id falls back to doc_id.",
    )
    ap.add_argument(
        "--source_col",
        type=str,
        default="source",
        help="Optional column name for corpus tag (e.g., 'wos', 'arxiv').",
    )

    return ap.parse_args()

def main() -> None:
    args = parse_args()

    meta_path = Path(args.metadata_file)
    if not meta_path.is_file():
        raise FileNotFoundError(f"metadata_file does not exist or is not a file: {meta_path}")

    metas = load_metadata(
        meta_path,
        id_col=args.id_col,
        field_col=args.field_col,
        subfield_col=args.subfield_col,
        discipline_col=args.discipline_col,
        text_col=args.text_col,
        pdf_path_col=args.pdf_path_col or None,
        source_col=args.source_col or None,
    )
    if not metas:
        raise RuntimeError(f"No valid paper records were loaded from metadata file: {meta_path}")

    field_ids, subfield_ids, discipline_ids = build_ontology(metas)

    spans = build_span_records(
        metas=metas,
        field_ids=field_ids,
        subfield_ids=subfield_ids,
        discipline_ids=discipline_ids,
    )

    spans_out = Path(args.spans_out)
    spans_out.parent.mkdir(parents=True, exist_ok=True)
    write_spans_jsonl(spans, spans_out)

    if args.nodes_out:
        nodes_out = Path(args.nodes_out)
        nodes_out.parent.mkdir(parents=True, exist_ok=True)
        write_nodes_jsonl(nodes_out, field_ids, subfield_ids, discipline_ids)

    if args.level_sizes_out:
        ls_out = Path(args.level_sizes_out)
        ls_out.parent.mkdir(parents=True, exist_ok=True)
        write_level_sizes(field_ids, subfield_ids, discipline_ids, ls_out)

if __name__ == "__main__":
    main()


PaperMeta

    which you can copy into your training config as `level_sizes`.

The intended use is:
  1) Prepare a single metadata file that combines your WOS and arXiv papers,
     mapped into a 3-level taxonomy (field → subfield → discipline).
  2) Point this script at it to obtain spans and ontology metadata compatible
     with the rest of the TOLBERT pipeline.
"""

from __future__ import annotations

import argparse
import csv
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, List, Optional, Tuple

@dataclass
class PaperMeta:
    doc_id: str
    field: str
    subfield: str
    discipline: str
    text: str
    pdf_path: Optional[str] = None
    source: Optional[str] = None

def load_metadata(
    path: Path,
    *,
    id_col: str,
    field_col: str,
    subfield_col: str,
    discipline_col: str,
    text_col: str,
    pdf_path_col: Optional[str],
    source_col: Optional[str],
) -> List[PaperMeta]:
    """
    Load WOS / arXiv-style paper metadata from CSV or JSON(L).

    The loader is column-name agnostic: you can override the column names via CLI.
    """
    metas: List[PaperMeta] = []


import argparse
import csv
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, List, Optional, Tuple

@dataclass
class PaperMeta:
    doc_id: str
    field: str
    subfield: str
    discipline: str
    text: str
    pdf_path: Optional[str] = None
    source: Optional[str] = None

def load_metadata(
    path: Path,
    *,
    id_col: str,
    field_col: str,
    subfield_col: str,
    discipline_col: str,
    text_col: str,
    pdf_path_col: Optional[str],
    source_col: Optional[str],
) -> List[PaperMeta]:
    """
    Load WOS / arXiv-style paper metadata from CSV or JSON(L).

    The loader is column-name agnostic: you can override the column names via CLI.
    """
    metas: List[PaperMeta] = []

    def _row_to_meta(row: Dict[str, object]) -> Optional[PaperMeta]:
        try:
            doc_id = str(row[id_col])
            field = str(row[field_col])
            subfield = str(row[subfield_col])
            discipline = str(row[discipline_col])
            text = str(row[text_col])
        except KeyError as exc:
            raise KeyError(f"Missing required column {exc!s} in metadata record: {row}") from exc

        if not text.strip():
            # Skip empty-text entries; they are not useful training instances.
            return None

        pdf_path_val: Optional[str] = None
        if pdf_path_col and pdf_path_col in row and row[pdf_path_col] is not None:
            pdf_path_val = str(row[pdf_path_col])

        source_val: Optional[str] = None
        if source_col and source_col in row and row[source_col] is not None:
            source_val = str(row[source_col])

        return PaperMeta(
            doc_id=doc_id,
            field=field,
            subfield=subfield,
            discipline=discipline,
            text=text,
            pdf_path=pdf_path_val,
            source=source_val,
        )

    suffix = path.suffix.lower()
    if suffix in {".jsonl", ".json"}:
        with path.open("r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                obj = json.loads(line)
                # Accept either a flat object or nested under "data"
                if isinstance(obj, dict) and "data" in obj and isinstance(obj["data"], dict):
                    obj = obj["data"]
                if not isinstance(obj, dict):
                    continue
                meta = _row_to_meta(obj)
                if meta is not None:
                    metas.append(meta)
        return metas

    # Default: CSV with header.
    with path.open("r", encoding="utf-8") as f:
        reader = csv.DictReader(f)
        for row in reader:
            meta = _row_to_meta(row)
            if meta is not None:
                metas.append(meta)
    return metas

def build_ontology(
    metas: List[PaperMeta],
) -> Tuple[Dict[str, int], Dict[str, int], Dict[str, int]]:
    """
    Build simple integer ID mappings:
      - field -> field_node_id
      - (field, subfield) -> subfield_node_id
      - (field, subfield, discipline) -> discipline_node_id

    Level conventions:
      level 0: root (id 0)
      level 1: field nodes
      level 2: subfield nodes
      level 3: discipline nodes
    """
    next_id = 0
    root_id = next_id
    next_id += 1


                # Accept either a flat object or nested under "data"
                if isinstance(obj, dict) and "data" in obj and isinstance(obj["data"], dict):
                    obj = obj["data"]
                if not isinstance(obj, dict):
                    continue
                meta = _row_to_meta(obj)
                if meta is not None:
                    metas.append(meta)
        return metas

    # Default: CSV with header.
    with path.open("r", encoding="utf-8") as f:
        reader = csv.DictReader(f)
        for row in reader:
            meta = _row_to_meta(row)
            if meta is not None:
                metas.append(meta)
    return metas

def build_ontology(
    metas: List[PaperMeta],
) -> Tuple[Dict[str, int], Dict[str, int], Dict[str, int]]:
    """
    Build simple integer ID mappings:
      - field -> field_node_id
      - (field, subfield) -> subfield_node_id
      - (field, subfield, discipline) -> discipline_node_id

    Level conventions:
      level 0: root (id 0)
      level 1: field nodes
      level 2: subfield nodes
      level 3: discipline nodes
    """
    next_id = 0
    root_id = next_id
    next_id += 1

    field_ids: Dict[str, int] = {}
    subfield_ids: Dict[str, int] = {}
    discipline_ids: Dict[str, int] = {}

    for m in metas:
        if m.field not in field_ids:
            field_ids[m.field] = next_id
            next_id += 1

        sub_key = f"{m.field}::{m.subfield}"
        if sub_key not in subfield_ids:
            subfield_ids[sub_key] = next_id
            next_id += 1

        disc_key = f"{m.field}::{m.subfield}::{m.discipline}"
        if disc_key not in discipline_ids:
            discipline_ids[disc_key] = next_id
            next_id += 1

    # Sanity check: at least one node at each level if any metas were provided.
    if metas and (not field_ids or not subfield_ids or not discipline_ids):
        raise RuntimeError("Failed to build a non-empty 3-level ontology from metadata.")

    # root_id is currently unused in the returned dicts but kept for documentation.
    _ = root_id
    return field_ids, subfield_ids, discipline_ids

def write_nodes_jsonl(
    out_path: Path,
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
) -> None:
    """
    Emit a minimal nodes.jsonl compatible with docs/tree_of_life.md.
    """
    with out_path.open("w", encoding="utf-8") as f:
        # Root (level 0)
        f.write(
            json.dumps(
                {
                    "node_id": 0,
                    "level": 0,
                    "type": "root",
                    "parent_id": None,


        sub_key = f"{m.field}::{m.subfield}"
        if sub_key not in subfield_ids:
            subfield_ids[sub_key] = next_id
            next_id += 1

        disc_key = f"{m.field}::{m.subfield}::{m.discipline}"
        if disc_key not in discipline_ids:
            discipline_ids[disc_key] = next_id
            next_id += 1

    # Sanity check: at least one node at each level if any metas were provided.
    if metas and (not field_ids or not subfield_ids or not discipline_ids):
        raise RuntimeError("Failed to build a non-empty 3-level ontology from metadata.")

    # root_id is currently unused in the returned dicts but kept for documentation.
    _ = root_id
    return field_ids, subfield_ids, discipline_ids

def write_nodes_jsonl(
    out_path: Path,
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
) -> None:
    """
    Emit a minimal nodes.jsonl compatible with docs/tree_of_life.md.
    """
    with out_path.open("w", encoding="utf-8") as f:
        # Root (level 0)
        f.write(
            json.dumps(
                {
                    "node_id": 0,
                    "level": 0,
                    "type": "root",
                    "parent_id": None,
                    "name": "Root",
                    "attributes": {},
                }
            )
            + "\n"
        )

        # Fields (level 1)
        for field_name, nid in field_ids.items():
            f.write(
                json.dumps(
                    {
                        "node_id": nid,
                        "level": 1,
                        "type": "field",
                        "parent_id": 0,
                        "name": field_name,
                        "attributes": {},
                    }
                )
                + "\n"
            )

        # Subfields (level 2)
        for key, nid in subfield_ids.items():
            field_name, subfield_name = key.split("::", 1)
            parent_id = field_ids[field_name]
            f.write(
                json.dumps(
                    {
                        "node_id": nid,
                        "level": 2,
                        "type": "subfield",
                        "parent_id": parent_id,
                        "name": subfield_name,
                        "attributes": {"field": field_name},
                    }
                )
                + "\n"
            )

        # Disciplines (level 3)
        for key, nid in discipline_ids.items():
            field_name, subfield_name, discipline_name = key.split("::", 2)
            sub_key = f"{field_name}::{subfield_name}"
            parent_id = subfield_ids[sub_key]
            f.write(
                json.dumps(
                    {
                        "node_id": nid,
                        "level": 3,
                        "type": "discipline",
                        "parent_id": parent_id,
                        "name": discipline_name,
                        "attributes": {
                            "field": field_name,
                            "subfield": subfield_name,
                        },
                    }
                )
                + "\n"
            )

def build_span_records(
    metas: List[PaperMeta],
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
) -> List[Dict[str, object]]:
    """
    Iterate over papers and generate one span per paper.
    """
    root_id = 0
    spans: List[Dict[str, object]] = []

    for m in metas:
        field_id = field_ids[m.field]
        sub_key = f"{m.field}::{m.subfield}"
        sub_id = subfield_ids[sub_key]
        disc_key = f"{m.field}::{m.subfield}::{m.discipline}"
        disc_id = discipline_ids[disc_key]


            sub_key = f"{field_name}::{subfield_name}"
            parent_id = subfield_ids[sub_key]
            f.write(
                json.dumps(
                    {
                        "node_id": nid,
                        "level": 3,
                        "type": "discipline",
                        "parent_id": parent_id,
                        "name": discipline_name,
                        "attributes": {
                            "field": field_name,
                            "subfield": subfield_name,
                        },
                    }
                )
                + "\n"
            )

def build_span_records(
    metas: List[PaperMeta],
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
) -> List[Dict[str, object]]:
    """
    Iterate over papers and generate one span per paper.
    """
    root_id = 0
    spans: List[Dict[str, object]] = []

    for m in metas:
        field_id = field_ids[m.field]
        sub_key = f"{m.field}::{m.subfield}"
        sub_id = subfield_ids[sub_key]
        disc_key = f"{m.field}::{m.subfield}::{m.discipline}"
        disc_id = discipline_ids[disc_key]

        node_path = [root_id, field_id, sub_id, disc_id]

        span_id = m.doc_id
        source_id = m.pdf_path if m.pdf_path is not None else m.doc_id

        meta: Dict[str, object] = {
            "doc_id": m.doc_id,
            "field": m.field,
            "subfield": m.subfield,
            "discipline": m.discipline,
        }
        if m.pdf_path is not None:
            meta["pdf_path"] = m.pdf_path
        if m.source is not None:
            meta["source"] = m.source

        spans.append(
            {
                "span_id": span_id,
                "text": m.text,
                "source_id": source_id,
                "node_path": node_path,
                "meta": meta,
            }
        )

    return spans

def write_spans_jsonl(spans: List[Dict[str, object]], out_path: Path) -> None:
    with out_path.open("w", encoding="utf-8") as f:
        for rec in spans:
            f.write(json.dumps(rec) + "\n")

def write_level_sizes(
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
    out_path: Path,
) -> None:
    # We always have exactly one root.
    level_sizes = [
        1,  # root
        len(field_ids),
        len(subfield_ids),
        len(discipline_ids),


            "discipline": m.discipline,
        }
        if m.pdf_path is not None:
            meta["pdf_path"] = m.pdf_path
        if m.source is not None:
            meta["source"] = m.source

        spans.append(
            {
                "span_id": span_id,
                "text": m.text,
                "source_id": source_id,
                "node_path": node_path,
                "meta": meta,
            }
        )

    return spans

def write_spans_jsonl(spans: List[Dict[str, object]], out_path: Path) -> None:
    with out_path.open("w", encoding="utf-8") as f:
        for rec in spans:
            f.write(json.dumps(rec) + "\n")

def write_level_sizes(
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
    out_path: Path,
) -> None:
    # We always have exactly one root.
    level_sizes = [
        1,  # root
        len(field_ids),
        len(subfield_ids),
        len(discipline_ids),
    ]
    out = {"level_sizes": level_sizes}
    with out_path.open("w", encoding="utf-8") as f:
        json.dump(out, f, indent=2)


        spans.append(
            {
                "span_id": span_id,
                "text": m.text,
                "source_id": source_id,
                "node_path": node_path,
                "meta": meta,
            }
        )

    return spans

def write_spans_jsonl(spans: List[Dict[str, object]], out_path: Path) -> None:
    with out_path.open("w", encoding="utf-8") as f:
        for rec in spans:
            f.write(json.dumps(rec) + "\n")

def write_level_sizes(
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
    out_path: Path,
) -> None:
    # We always have exactly one root.
    level_sizes = [
        1,  # root
        len(field_ids),
        len(subfield_ids),
        len(discipline_ids),
    ]
    out = {"level_sizes": level_sizes}
    with out_path.open("w", encoding="utf-8") as f:
        json.dump(out, f, indent=2)

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(
        description=(
            "Build ResearchHierarchy spans JSONL and ontology metadata from WOS / arXiv metadata."
        )
    )
    ap.add_argument(
        "--metadata_file",
        type=str,
        required=True,
        help="CSV or JSONL file with at least: doc_id, field, subfield, discipline, text.",
    )
    ap.add_argument(
        "--spans_out",
        type=str,
        required=True,
        help="Output path for spans JSONL file.",
    )


def write_level_sizes(
    field_ids: Dict[str, int],
    subfield_ids: Dict[str, int],
    discipline_ids: Dict[str, int],
    out_path: Path,
) -> None:
    # We always have exactly one root.
    level_sizes = [
        1,  # root
        len(field_ids),
        len(subfield_ids),
        len(discipline_ids),
    ]
    out = {"level_sizes": level_sizes}
    with out_path.open("w", encoding="utf-8") as f:
        json.dump(out, f, indent=2)

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(
        description=(
            "Build ResearchHierarchy spans JSONL and ontology metadata from WOS / arXiv metadata."
        )
    )
    ap.add_argument(
        "--metadata_file",
        type=str,
        required=True,
        help="CSV or JSONL file with at least: doc_id, field, subfield, discipline, text.",
    )
    ap.add_argument(
        "--spans_out",
        type=str,
        required=True,
        help="Output path for spans JSONL file.",
    )
    ap.add_argument(
        "--nodes_out",
        type=str,
        default="",
        help="Optional output path for nodes JSONL (ontology nodes).",
    )
    ap.add_argument(
        "--level_sizes_out",
        type=str,
        default="",
        help="Optional output path for level_sizes JSON file.",
    )

    # Column name overrides for flexibility.
    ap.add_argument("--id_col", type=str, default="doc_id", help="Column name for document ID.")
    ap.add_argument("--field_col", type=str, default="field", help="Column name for level-1 field.")
    ap.add_argument(
        "--subfield_col",
        type=str,
        default="subfield",
        help="Column name for level-2 subfield.",
    )
    ap.add_argument(
        "--discipline_col",
        type=str,
        default="discipline",
        help="Column name for level-3 discipline.",
    )
    ap.add_argument(
        "--text_col",
        type=str,
        default="text",
        help="Column name containing the paper text (e.g., title + abstract).",
    )
    ap.add_argument(
        "--pdf_path_col",
        type=str,
        default="pdf_path",
        help="Optional column name for the PDF path; if missing, source_id falls back to doc_id.",
    )
    ap.add_argument(
        "--source_col",
        type=str,
        default="source",
        help="Optional column name for corpus tag (e.g., 'wos', 'arxiv').",
    )

    return ap.parse_args()

def main() -> None:
    args = parse_args()

    meta_path = Path(args.metadata_file)
    if not meta_path.is_file():
        raise FileNotFoundError(f"metadata_file does not exist or is not a file: {meta_path}")

    metas = load_metadata(
        meta_path,
        id_col=args.id_col,
        field_col=args.field_col,
        subfield_col=args.subfield_col,
        discipline_col=args.discipline_col,
        text_col=args.text_col,
        pdf_path_col=args.pdf_path_col or None,
        source_col=args.source_col or None,
    )
    if not metas:


main

        type=str,
        default="text",
        help="Column name containing the paper text (e.g., title + abstract).",
    )
    ap.add_argument(
        "--pdf_path_col",
        type=str,
        default="pdf_path",
        help="Optional column name for the PDF path; if missing, source_id falls back to doc_id.",
    )
    ap.add_argument(
        "--source_col",
        type=str,
        default="source",
        help="Optional column name for corpus tag (e.g., 'wos', 'arxiv').",
    )

    return ap.parse_args()

def main() -> None:
    args = parse_args()

    meta_path = Path(args.metadata_file)
    if not meta_path.is_file():
        raise FileNotFoundError(f"metadata_file does not exist or is not a file: {meta_path}")

    metas = load_metadata(
        meta_path,
        id_col=args.id_col,
        field_col=args.field_col,
        subfield_col=args.subfield_col,
        discipline_col=args.discipline_col,
        text_col=args.text_col,
        pdf_path_col=args.pdf_path_col or None,
        source_col=args.source_col or None,
    )
    if not metas:
        raise RuntimeError(f"No valid paper records were loaded from metadata file: {meta_path}")

    field_ids, subfield_ids, discipline_ids = build_ontology(metas)

    spans = build_span_records(
        metas=metas,
        field_ids=field_ids,
        subfield_ids=subfield_ids,
        discipline_ids=discipline_ids,
    )

    spans_out = Path(args.spans_out)
    spans_out.parent.mkdir(parents=True, exist_ok=True)
    write_spans_jsonl(spans, spans_out)

    if args.nodes_out:
        nodes_out = Path(args.nodes_out)
        nodes_out.parent.mkdir(parents=True, exist_ok=True)
        write_nodes_jsonl(nodes_out, field_ids, subfield_ids, discipline_ids)

    if args.level_sizes_out:
        ls_out = Path(args.level_sizes_out)
        ls_out.parent.mkdir(parents=True, exist_ok=True)
        write_level_sizes(field_ids, subfield_ids, discipline_ids, ls_out)

if __name__ == "__main__":
    main()


def load_metadata(
    path: Path,
    *,
    id_col: str,
    field_col: str,
    subfield_col: str,
    discipline_col: str,
    text_col: str,
    pdf_path_col: Optional[str],
    source_col: Optional[str],
) -> List[PaperMeta]:
    """
    Load WOS / arXiv-style paper metadata from CSV or JSON(L).

    The loader is column-name agnostic: you can override the column names via CLI.
    """
    metas: List[PaperMeta] = []

    def _row_to_meta(row: Dict[str, object]) -> Optional[PaperMeta]:
        try:
            doc_id = str(row[id_col])
            field = str(row[field_col])
            subfield = str(row[subfield_col])
            discipline = str(row[discipline_col])
            text = str(row[text_col])
        except KeyError as exc:
            raise KeyError(f"Missing required column {exc!s} in metadata record: {row}") from exc

        if not text.strip():
            # Skip empty-text entries; they are not useful training instances.
            return None

        pdf_path_val: Optional[str] = None
        if pdf_path_col and pdf_path_col in row and row[pdf_path_col] is not None:
            pdf_path_val = str(row[pdf_path_col])

        source_val: Optional[str] = None
        if source_col and source_col in row and row[source_col] is not None:
            source_val = str(row[source_col])

        return PaperMeta(
            doc_id=doc_id,
            field=field,
            subfield=subfield,
            discipline=discipline,
            text=text,
            pdf_path=pdf_path_val,
            source=source_val,
        )

    suffix = path.suffix.lower()
    if suffix in {".jsonl", ".json"}:
        with path.open("r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                obj = json.loads(line)
                # Accept either a flat object or nested under "data"
                if isinstance(obj, dict) and "data" in obj and isinstance(obj["data"], dict):
                    obj = obj["data"]
                if not isinstance(obj, dict):
                    continue
                meta = _row_to_meta(obj)
                if meta is not None:
                    metas.append(meta)
        return metas

    # Default: CSV with header.


"""
Agent / LLM stack skeleton: PR -> paper recommendations via TOLBERT.

This script is a high-level wiring of the "worked scenario" from `docs/usage.md`:
  1. Ingest a pull request (PR) as text spans.
  2. Encode spans with TOLBERT.
  3. Locate the PR within the Tree-of-Life (domain/subdomain).
  4. Retrieve relevant paper spans from a pre-built index.
  5. Use an LLM to summarize / explain the recommendations.

You are expected to plug in:
  - real PR ingestion,
  - a proper paper-span index (vector DB, search service, etc.),
  - and your own LLM backend in `call_llm`.
"""

import argparse
from pathlib import Path
from typing import Any, Dict, List, Tuple

import torch
from torch.nn import functional as F
from transformers import AutoTokenizer
import os

from tolbert.config import load_tolbert_config
from tolbert.modeling import TOLBERT, TOLBERTConfig

def call_llm(prompt: str) -> str:
    """
    Placeholder for your LLM call.

    Replace this with your own LLM client (OpenAI, local model, etc.).
    """
    raise NotImplementedError("Integrate your LLM client here.")

def build_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def simple_pr_spans(pr_text: str) -> List[str]:
    """
    Very simple PR spanizer:
      - splits on double newlines,
      - keeps reasonably sized chunks as candidate spans.

    You likely want to replace this with something that understands diffs
    and function boundaries.
    """
    parts = [p.strip() for p in pr_text.split("\n\n") if p.strip()]
    return parts

def encode_spans(
    model: TOLBERT,
    tokenizer,
    spans: List[str],
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    embs: List[torch.Tensor] = []
    metas: List[Dict[str, Any]] = []
    for i, text in enumerate(spans):
        tokens = tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        with torch.no_grad():
            out = model(
                input_ids=tokens["input_ids"].to(device),
                attention_mask=tokens["attention_mask"].to(device),
            )
        embs.append(out["proj"].squeeze(0).cpu())

        # Record predicted hierarchy information for analysis.
        level_logits = out["level_logits"]
        level_preds = {
            int(level): logits.argmax(dim=-1).item()
            for level, logits in level_logits.items()
        }

        metas.append(
            {
                "span_index": i,
                "text": text,
                "level_predictions": level_preds,
            }
        )

    if not embs:
        return torch.empty(0), metas
    return torch.stack(embs, dim=0), metas

def load_paper_index(index_path: str) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Load a pre-built paper-span index.

    This function is intentionally schematic. You might, for example, store:
      - embeddings as a `.pt` tensor,
      - metadata as a JSON/JSONL file with the same length.
    """
    index_dir = Path(index_path)
    emb_path = index_dir / "embeddings.pt"
    meta_path = index_dir / "metadata.jsonl"

    if not emb_path.exists() or not meta_path.exists():
        raise FileNotFoundError(
            f"Expected embeddings at {emb_path} and metadata at {meta_path}."
        )

    embs = torch.load(emb_path, map_location="cpu")
    metas: List[Dict[str, Any]] = []
    with meta_path.open("r", encoding="utf-8") as f:
        import json

        for line in f:
            line = line.strip()
            if not line:
                continue
            metas.append(json.loads(line))

    if embs.size(0) != len(metas):
        raise ValueError("Mismatch between number of embeddings and metadata entries.")

    return embs, metas

def retrieve_top_papers(
    pr_embs: torch.Tensor,
    paper_embs: torch.Tensor,
    paper_metas: List[Dict[str, Any]],
    top_k_per_span: int = 5,
    top_k_papers: int = 10,
) -> List[Dict[str, Any]]:
    """
    Simple multi-span retrieval:
      - for each PR span embedding, retrieve top_k_per_span paper spans,
      - aggregate scores per paper id,
      - return top_k_papers papers with aggregated scores.
    """
    if pr_embs.numel() == 0 or paper_embs.numel() == 0:
        return []

    scores: Dict[str, float] = {}
    for i in range(pr_embs.size(0)):
        q = pr_embs[i : i + 1]
        sims = F.cosine_similarity(q, paper_embs, dim=1)
        topk = torch.topk(sims, k=min(top_k_per_span, sims.numel()))
        for idx, score in zip(topk.indices.tolist(), topk.values.tolist()):
            meta = paper_metas[idx]
            paper_id = str(meta.get("paper_id", idx))
            scores[paper_id] = scores.get(paper_id, 0.0) + float(score)

    ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)[:top_k_papers]
    results: List[Dict[str, Any]] = []
    for paper_id, score in ranked:
        # find first meta entry with this paper_id
        meta = next((m for m in paper_metas if str(m.get("paper_id")) == paper_id), None)
        if meta is None:
            continue
        results.append(
            {
                "paper_id": paper_id,
                "score": score,
                "title": meta.get("title"),
                "abstract": meta.get("abstract"),
            }
        )
    return results

def build_prompt(pr_text: str, recommendations: List[Dict[str, Any]]) -> str:
    """
    Construct a prompt for the LLM given:
      - the PR text,
      - a set of candidate papers with scores and metadata.
    """
    lines = [
        "You are an expert AI code reviewer and research assistant.",
        "Given the following pull request and a set of candidate research papers,",
        "explain which papers are most relevant and why.",
        "",
        "PULL REQUEST:",
        pr_text,
        "",
        "CANDIDATE PAPERS:",
    ]
    for i, rec in enumerate(recommendations, start=1):
        lines.append(
            f"{i}. [paper_id={rec['paper_id']}] title={rec.get('title')!r} "
            f"(score={rec['score']:.3f})"
        )
        if rec.get("abstract"):
            lines.append(f"   abstract={rec['abstract']!r}")
    lines.append("")
    lines.append(
        "For each paper above, give a short explanation (2-4 sentences) "
        "of how it relates to the PR, and suggest how the PR could benefit "
        "from ideas in these papers."
    )
    return "\n".join(lines)

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="PR -> paper recommendation agent (skeleton).")
    ap.add_argument("--config", type=str, required=True, help="Path to TOLBERT config.")
    ap.add_argument("--checkpoint", type=str, required=True, help="Path to model checkpoint (.pt).")
    ap.add_argument(
        "--paper_index",
        type=str,
        required=True,
        help="Path to directory with paper-span embeddings and metadata.",
    )
    ap.add_argument(
        "--pr_file",
        type=str,
        required=True,
        help="Path to a text file with PR description / diff.",
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)
    device = torch.device(args.device)

    checkpoint_path = Path(args.checkpoint)
    if not checkpoint_path.exists():
        raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}")

    tokenizer = AutoTokenizer.from_pretrained(
        cfg["base_model_name"],
        cache_dir="/data/checkpoints/",  # noqa: E501
    )
    model = build_model(cfg, str(checkpoint_path), device=device)

    # Load and span-ize the PR
    pr_text = Path(args.pr_file).read_text(encoding="utf-8")
    pr_spans = simple_pr_spans(pr_text)

    pr_embs, span_metas = encode_spans(
        model=model,
        tokenizer=tokenizer,
        spans=pr_spans,
        max_length=cfg.get("max_length", 256),
        device=device,
    )

    # Load paper index and retrieve recommendations
    paper_embs, paper_metas = load_paper_index(args.paper_index)
    recommendations = retrieve_top_papers(
        pr_embs=pr_embs,
        paper_embs=paper_embs,
        paper_metas=paper_metas,
        top_k_per_span=cfg.get("top_k_per_span", 5),
        top_k_papers=cfg.get("top_k_papers", 10),
    )

    if not recommendations:
        print("No paper recommendations found (empty embeddings or index).")
        return

    prompt = build_prompt(pr_text, recommendations)

    # Hand off to LLM (you implement this)
    print("=== LLM PROMPT BEGIN ===")
    print(prompt)
    print("=== LLM PROMPT END ===")
    print()
    print("Now call your LLM client with the prompt above (see `call_llm`).")

if __name__ == "__main__":
    main()


You are expected to plug in:
  - real PR ingestion,
  - a proper paper-span index (vector DB, search service, etc.),
  - and your own LLM backend in `call_llm`.
"""

import argparse
from pathlib import Path
from typing import Any, Dict, List, Tuple

import torch
from torch.nn import functional as F
from transformers import AutoTokenizer
import os

from tolbert.config import load_tolbert_config
from tolbert.modeling import TOLBERT, TOLBERTConfig

def call_llm(prompt: str) -> str:
    """
    Placeholder for your LLM call.

    Replace this with your own LLM client (OpenAI, local model, etc.).
    """
    raise NotImplementedError("Integrate your LLM client here.")

def build_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def simple_pr_spans(pr_text: str) -> List[str]:
    """
    Very simple PR spanizer:
      - splits on double newlines,


from typing import Any, Dict, List, Tuple

import torch
from torch.nn import functional as F
from transformers import AutoTokenizer
import os

from tolbert.config import load_tolbert_config
from tolbert.modeling import TOLBERT, TOLBERTConfig

def call_llm(prompt: str) -> str:
    """
    Placeholder for your LLM call.

    Replace this with your own LLM client (OpenAI, local model, etc.).
    """
    raise NotImplementedError("Integrate your LLM client here.")

def build_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def simple_pr_spans(pr_text: str) -> List[str]:
    """
    Very simple PR spanizer:
      - splits on double newlines,
      - keeps reasonably sized chunks as candidate spans.

    You likely want to replace this with something that understands diffs
    and function boundaries.
    """
    parts = [p.strip() for p in pr_text.split("\n\n") if p.strip()]
    return parts

def encode_spans(
    model: TOLBERT,
    tokenizer,
    spans: List[str],
    max_length: int,


    Replace this with your own LLM client (OpenAI, local model, etc.).
    """
    raise NotImplementedError("Integrate your LLM client here.")

def build_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def simple_pr_spans(pr_text: str) -> List[str]:
    """
    Very simple PR spanizer:
      - splits on double newlines,
      - keeps reasonably sized chunks as candidate spans.

    You likely want to replace this with something that understands diffs
    and function boundaries.
    """
    parts = [p.strip() for p in pr_text.split("\n\n") if p.strip()]
    return parts

def encode_spans(
    model: TOLBERT,
    tokenizer,
    spans: List[str],
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    embs: List[torch.Tensor] = []
    metas: List[Dict[str, Any]] = []
    for i, text in enumerate(spans):
        tokens = tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        with torch.no_grad():


    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def simple_pr_spans(pr_text: str) -> List[str]:
    """
    Very simple PR spanizer:
      - splits on double newlines,
      - keeps reasonably sized chunks as candidate spans.

    You likely want to replace this with something that understands diffs
    and function boundaries.
    """
    parts = [p.strip() for p in pr_text.split("\n\n") if p.strip()]
    return parts

def encode_spans(
    model: TOLBERT,
    tokenizer,
    spans: List[str],
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    embs: List[torch.Tensor] = []
    metas: List[Dict[str, Any]] = []
    for i, text in enumerate(spans):
        tokens = tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        with torch.no_grad():
            out = model(
                input_ids=tokens["input_ids"].to(device),
                attention_mask=tokens["attention_mask"].to(device),
            )
        embs.append(out["proj"].squeeze(0).cpu())

        # Record predicted hierarchy information for analysis.
        level_logits = out["level_logits"]
        level_preds = {
            int(level): logits.argmax(dim=-1).item()
            for level, logits in level_logits.items()
        }

        metas.append(
            {
                "span_index": i,
                "text": text,
                "level_predictions": level_preds,
            }
        )

    if not embs:
        return torch.empty(0), metas
    return torch.stack(embs, dim=0), metas

def load_paper_index(index_path: str) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Load a pre-built paper-span index.

    This function is intentionally schematic. You might, for example, store:
      - embeddings as a `.pt` tensor,
      - metadata as a JSON/JSONL file with the same length.
    """
    index_dir = Path(index_path)
    emb_path = index_dir / "embeddings.pt"
    meta_path = index_dir / "metadata.jsonl"

    if not emb_path.exists() or not meta_path.exists():
        raise FileNotFoundError(
            f"Expected embeddings at {emb_path} and metadata at {meta_path}."
        )

    embs = torch.load(emb_path, map_location="cpu")


        # Record predicted hierarchy information for analysis.
        level_logits = out["level_logits"]
        level_preds = {
            int(level): logits.argmax(dim=-1).item()
            for level, logits in level_logits.items()
        }

        metas.append(
            {
                "span_index": i,
                "text": text,
                "level_predictions": level_preds,
            }
        )

    if not embs:
        return torch.empty(0), metas
    return torch.stack(embs, dim=0), metas

def load_paper_index(index_path: str) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Load a pre-built paper-span index.

    This function is intentionally schematic. You might, for example, store:
      - embeddings as a `.pt` tensor,
      - metadata as a JSON/JSONL file with the same length.
    """
    index_dir = Path(index_path)
    emb_path = index_dir / "embeddings.pt"
    meta_path = index_dir / "metadata.jsonl"

    if not emb_path.exists() or not meta_path.exists():
        raise FileNotFoundError(
            f"Expected embeddings at {emb_path} and metadata at {meta_path}."
        )

    embs = torch.load(emb_path, map_location="cpu")
    metas: List[Dict[str, Any]] = []
    with meta_path.open("r", encoding="utf-8") as f:
        import json

        for line in f:
            line = line.strip()
            if not line:
                continue
            metas.append(json.loads(line))

    if embs.size(0) != len(metas):
        raise ValueError("Mismatch between number of embeddings and metadata entries.")

    return embs, metas

def retrieve_top_papers(
    pr_embs: torch.Tensor,
    paper_embs: torch.Tensor,
    paper_metas: List[Dict[str, Any]],
    top_k_per_span: int = 5,
    top_k_papers: int = 10,
) -> List[Dict[str, Any]]:
    """
    Simple multi-span retrieval:
      - for each PR span embedding, retrieve top_k_per_span paper spans,
      - aggregate scores per paper id,
      - return top_k_papers papers with aggregated scores.
    """
    if pr_embs.numel() == 0 or paper_embs.numel() == 0:
        return []

    scores: Dict[str, float] = {}
    for i in range(pr_embs.size(0)):


            f"Expected embeddings at {emb_path} and metadata at {meta_path}."
        )

    embs = torch.load(emb_path, map_location="cpu")
    metas: List[Dict[str, Any]] = []
    with meta_path.open("r", encoding="utf-8") as f:
        import json

        for line in f:
            line = line.strip()
            if not line:
                continue
            metas.append(json.loads(line))

    if embs.size(0) != len(metas):
        raise ValueError("Mismatch between number of embeddings and metadata entries.")

    return embs, metas

def retrieve_top_papers(
    pr_embs: torch.Tensor,
    paper_embs: torch.Tensor,
    paper_metas: List[Dict[str, Any]],
    top_k_per_span: int = 5,
    top_k_papers: int = 10,
) -> List[Dict[str, Any]]:
    """
    Simple multi-span retrieval:
      - for each PR span embedding, retrieve top_k_per_span paper spans,
      - aggregate scores per paper id,
      - return top_k_papers papers with aggregated scores.
    """
    if pr_embs.numel() == 0 or paper_embs.numel() == 0:
        return []

    scores: Dict[str, float] = {}
    for i in range(pr_embs.size(0)):
        q = pr_embs[i : i + 1]
        sims = F.cosine_similarity(q, paper_embs, dim=1)
        topk = torch.topk(sims, k=min(top_k_per_span, sims.numel()))
        for idx, score in zip(topk.indices.tolist(), topk.values.tolist()):
            meta = paper_metas[idx]
            paper_id = str(meta.get("paper_id", idx))
            scores[paper_id] = scores.get(paper_id, 0.0) + float(score)

    ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)[:top_k_papers]
    results: List[Dict[str, Any]] = []
    for paper_id, score in ranked:
        # find first meta entry with this paper_id
        meta = next((m for m in paper_metas if str(m.get("paper_id")) == paper_id), None)
        if meta is None:
            continue
        results.append(
            {
                "paper_id": paper_id,
                "score": score,
                "title": meta.get("title"),
                "abstract": meta.get("abstract"),
            }
        )
    return results

def build_prompt(pr_text: str, recommendations: List[Dict[str, Any]]) -> str:
    """
    Construct a prompt for the LLM given:
      - the PR text,
      - a set of candidate papers with scores and metadata.
    """
    lines = [
        "You are an expert AI code reviewer and research assistant.",
        "Given the following pull request and a set of candidate research papers,",
        "explain which papers are most relevant and why.",
        "",
        "PULL REQUEST:",
        pr_text,
        "",
        "CANDIDATE PAPERS:",
    ]
    for i, rec in enumerate(recommendations, start=1):
        lines.append(


            scores[paper_id] = scores.get(paper_id, 0.0) + float(score)

    ranked = sorted(scores.items(), key=lambda kv: kv[1], reverse=True)[:top_k_papers]
    results: List[Dict[str, Any]] = []
    for paper_id, score in ranked:
        # find first meta entry with this paper_id
        meta = next((m for m in paper_metas if str(m.get("paper_id")) == paper_id), None)
        if meta is None:
            continue
        results.append(
            {
                "paper_id": paper_id,
                "score": score,
                "title": meta.get("title"),
                "abstract": meta.get("abstract"),
            }
        )
    return results

def build_prompt(pr_text: str, recommendations: List[Dict[str, Any]]) -> str:
    """
    Construct a prompt for the LLM given:
      - the PR text,
      - a set of candidate papers with scores and metadata.
    """
    lines = [
        "You are an expert AI code reviewer and research assistant.",
        "Given the following pull request and a set of candidate research papers,",
        "explain which papers are most relevant and why.",
        "",
        "PULL REQUEST:",
        pr_text,
        "",
        "CANDIDATE PAPERS:",
    ]
    for i, rec in enumerate(recommendations, start=1):
        lines.append(
            f"{i}. [paper_id={rec['paper_id']}] title={rec.get('title')!r} "
            f"(score={rec['score']:.3f})"
        )
        if rec.get("abstract"):
            lines.append(f"   abstract={rec['abstract']!r}")
    lines.append("")
    lines.append(
        "For each paper above, give a short explanation (2-4 sentences) "
        "of how it relates to the PR, and suggest how the PR could benefit "
        "from ideas in these papers."
    )
    return "\n".join(lines)

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="PR -> paper recommendation agent (skeleton).")
    ap.add_argument("--config", type=str, required=True, help="Path to TOLBERT config.")
    ap.add_argument("--checkpoint", type=str, required=True, help="Path to model checkpoint (.pt).")
    ap.add_argument(
        "--paper_index",
        type=str,
        required=True,
        help="Path to directory with paper-span embeddings and metadata.",
    )
    ap.add_argument(
        "--pr_file",
        type=str,
        required=True,
        help="Path to a text file with PR description / diff.",
    )
    ap.add_argument(
        "--device",


        pr_text,
        "",
        "CANDIDATE PAPERS:",
    ]
    for i, rec in enumerate(recommendations, start=1):
        lines.append(
            f"{i}. [paper_id={rec['paper_id']}] title={rec.get('title')!r} "
            f"(score={rec['score']:.3f})"
        )
        if rec.get("abstract"):
            lines.append(f"   abstract={rec['abstract']!r}")
    lines.append("")
    lines.append(
        "For each paper above, give a short explanation (2-4 sentences) "
        "of how it relates to the PR, and suggest how the PR could benefit "
        "from ideas in these papers."
    )
    return "\n".join(lines)

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="PR -> paper recommendation agent (skeleton).")
    ap.add_argument("--config", type=str, required=True, help="Path to TOLBERT config.")
    ap.add_argument("--checkpoint", type=str, required=True, help="Path to model checkpoint (.pt).")
    ap.add_argument(
        "--paper_index",
        type=str,
        required=True,
        help="Path to directory with paper-span embeddings and metadata.",
    )
    ap.add_argument(
        "--pr_file",
        type=str,
        required=True,
        help="Path to a text file with PR description / diff.",
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)
    device = torch.device(args.device)

    checkpoint_path = Path(args.checkpoint)
    if not checkpoint_path.exists():
        raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}")

    tokenizer = AutoTokenizer.from_pretrained(
        cfg["base_model_name"],
        cache_dir="/data/checkpoints/",  # noqa: E501
    )
    model = build_model(cfg, str(checkpoint_path), device=device)

    # Load and span-ize the PR
    pr_text = Path(args.pr_file).read_text(encoding="utf-8")
    pr_spans = simple_pr_spans(pr_text)


        "--paper_index",
        type=str,
        required=True,
        help="Path to directory with paper-span embeddings and metadata.",
    )
    ap.add_argument(
        "--pr_file",
        type=str,
        required=True,
        help="Path to a text file with PR description / diff.",
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)
    device = torch.device(args.device)

    checkpoint_path = Path(args.checkpoint)
    if not checkpoint_path.exists():
        raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}")

    tokenizer = AutoTokenizer.from_pretrained(
        cfg["base_model_name"],
        cache_dir="/data/checkpoints/",  # noqa: E501
    )
    model = build_model(cfg, str(checkpoint_path), device=device)

    # Load and span-ize the PR
    pr_text = Path(args.pr_file).read_text(encoding="utf-8")
    pr_spans = simple_pr_spans(pr_text)

    pr_embs, span_metas = encode_spans(
        model=model,
        tokenizer=tokenizer,
        spans=pr_spans,
        max_length=cfg.get("max_length", 256),
        device=device,
    )

    # Load paper index and retrieve recommendations
    paper_embs, paper_metas = load_paper_index(args.paper_index)
    recommendations = retrieve_top_papers(
        pr_embs=pr_embs,
        paper_embs=paper_embs,
        paper_metas=paper_metas,
        top_k_per_span=cfg.get("top_k_per_span", 5),
        top_k_papers=cfg.get("top_k_papers", 10),
    )

    if not recommendations:
        print("No paper recommendations found (empty embeddings or index).")
        return

    prompt = build_prompt(pr_text, recommendations)

    # Hand off to LLM (you implement this)
    print("=== LLM PROMPT BEGIN ===")
    print(prompt)
    print("=== LLM PROMPT END ===")
    print()
    print("Now call your LLM client with the prompt above (see `call_llm`).")

if __name__ == "__main__":
    main()


"""
Build a joint code+paper Tree-of-Life (ToL) from existing per-domain spans.

This script is the missing glue described in the paper/docs: it takes
separate code and paper span files (each with its own local ontology and
`node_path` ids) and produces:

- A single, unified `nodes.jsonl` Tree-of-Life with:
    level 0: root
    level 1: domain nodes (e.g., "Code", "Papers")
    level 2+: remapped nodes from the original per-domain ontologies
- Rewritten span files for code and papers where `node_path` has been
  updated to refer to the *joint* node ID space.
- A `level_sizes` JSON helper compatible with `TOLBERTConfig.level_sizes`.

Design notes
============

This script deliberately **does not attempt to infer a semantic mapping**
between code categories (e.g., "Web", "ML", "Systems") and paper fields
("Computer Science", "Physics", ...). Doing that well would require
domain-specific heuristics or manual curation.

Instead, it:

- Adds explicit top-level domain nodes (by default: `"Code"` and `"Papers"`)
  at level 1 under a single shared root.
- Shifts each original ontology **one level deeper** in the tree:
    - Original root (id 0) is discarded; we introduce a fresh shared root.
    - Original level-1 nodes become level-2 under their domain node.
    - Original level-2 nodes become level-3, and so on.
- Remaps all node IDs into a **single global ID space** while preserving
  parent/child relations within each domain.

This gives you:

- A concrete, unified Tree-of-Life suitable for pretraining a single
  TOLBERT model across both domains.
- A starting point for more sophisticated, semantics-aware unification
  (e.g., manually or automatically clustering level-2 nodes across domains).

Inputs
======

- --code_spans:
    Spans JSONL for the code side (e.g., from `build_codehierarchy_spans.py`),
    with records of the form:
        {"span_id": "...", "text": "...", "source_id": "...", "node_path": [...], ...}

- --code_nodes:
    Ontology `nodes.jsonl` corresponding to `code_spans`. Expected fields:
        node_id, level, type, parent_id, name, attributes

- --paper_spans:
    Spans JSONL for the paper side (e.g., from `build_wos_spans.py` or
    `build_researchhierarchy_spans.py`).

- --paper_nodes:
    Ontology `nodes.jsonl` corresponding to `paper_spans`.

Outputs
=======

- --out_code_spans:
    Code spans JSONL with updated, joint `node_path`.

- --out_paper_spans:
    Paper spans JSONL with updated, joint `node_path`.

- --out_nodes:
    Joint ontology `nodes.jsonl` with a single root and shared ID space.

- --out_level_sizes:
    Small JSON file with:
        {"level_sizes": {1: num_level1, 2: num_level2, ...}}
    which you can plug into your TOLBERT training config.

Usage (example)
===============

1) Build per-domain datasets:

    python scripts/build_codehierarchy_spans.py \
        --repos_root /path/to/repos \
        --metadata_file /path/to/code_meta.csv \
        --spans_out data/code/spans_train.jsonl \
        --nodes_out data/code/nodes.jsonl

    python scripts/build_wos_spans.py \
        --input-csv /path/to/wos_train.csv \
        --spans-out data/wos/spans_train.jsonl \
        --nodes-out data/wos/nodes.jsonl

2) Join into a single Tree-of-Life:

    python scripts/build_joint_code_paper_tol.py \
        --code_spans data/code/spans_train.jsonl \
        --code_nodes data/code/nodes.jsonl \
        --paper_spans data/wos/spans_train.jsonl \
        --paper_nodes data/wos/nodes.jsonl \
        --out_code_spans data/joint/code_spans_train.jsonl \
        --out_paper_spans data/joint/paper_spans_train.jsonl \
        --out_nodes data/joint/nodes.jsonl \
        --out_level_sizes data/joint/level_sizes.json

You can then train TOLBERT on the union of `out_code_spans` and
`out_paper_spans` (e.g., via the `spans_files` list in your config) with
`level_sizes` taken from `out_level_sizes`.
"""

from __future__ import annotations

import argparse
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Tuple

@dataclass
class NodeRecord:
    node_id: int
    level: int
    type: str
    parent_id: int | None
    name: str
    attributes: Dict[str, object]

def _load_nodes(path: Path) -> Dict[int, NodeRecord]:
    nodes: Dict[int, NodeRecord] = {}
    with path.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            obj = json.loads(line)
            nid = int(obj["node_id"])
            level = int(obj.get("level", 0))
            node_type = str(obj.get("type", "node"))
            parent_id = obj.get("parent_id")
            if parent_id is not None:
                parent_id = int(parent_id)
            name = str(obj.get("name", f"node_{nid}"))
            attributes = obj.get("attributes") or {}
            nodes[nid] = NodeRecord(
                node_id=nid,
                level=level,
                type=node_type,
                parent_id=parent_id,
                name=name,
                attributes=attributes,
            )
    return nodes

def _iter_spans(path: Path) -> Iterable[Dict[str, object]]:
    with path.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            yield json.loads(line)

def _write_spans(path: Path, records: Iterable[Dict[str, object]]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

def _build_joint_ontology(
    *,
    code_nodes: Dict[int, NodeRecord],
    paper_nodes: Dict[int, NodeRecord],
    code_domain_name: str = "Code",
    paper_domain_name: str = "Papers",
) -> Tuple[
    Dict[int, NodeRecord],
    Dict[int, int],
    Dict[int, int],
    Dict[int, int],
]:
    """
    Construct a joint ontology and return:

    - joint_nodes: dict[new_id -> NodeRecord]
    - code_id_map: mapping old_code_node_id -> new_global_node_id
    - paper_id_map: mapping old_paper_node_id -> new_global_node_id
    - level_counts: mapping level -> number of nodes at that level (excluding root)

    Strategy:
      - Create a fresh root (id 0).
      - Create two domain nodes at level 1: Code, Papers.
      - For each original ontology:
          * Drop its root node (level 0).
          * Shift all other nodes one level deeper (level' = level + 1).
          * Rewire parents so that:
              - old parent == 0  → new parent = corresponding domain node id
              - otherwise        → new parent = mapped parent id
    """
    joint_nodes: Dict[int, NodeRecord] = {}
    level_counts: Dict[int, int] = {}

    # Root (level 0)
    root_id = 0
    joint_nodes[root_id] = NodeRecord(
        node_id=root_id,
        level=0,
        type="root",
        parent_id=None,
        name="Root",
        attributes={},
    )

    next_id = 1

    # Domain nodes at level 1
    code_domain_id = next_id
    next_id += 1
    paper_domain_id = next_id
    next_id += 1

    joint_nodes[code_domain_id] = NodeRecord(
        node_id=code_domain_id,
        level=1,
        type="domain",
        parent_id=root_id,
        name=code_domain_name,
        attributes={"source": "code"},
    )
    joint_nodes[paper_domain_id] = NodeRecord(
        node_id=paper_domain_id,
        level=1,
        type="domain",
        parent_id=root_id,
        name=paper_domain_name,
        attributes={"source": "paper"},
    )
    level_counts[1] = 2

    code_id_map: Dict[int, int] = {}
    paper_id_map: Dict[int, int] = {}

    def _remap_domain(
        *,
        src_nodes: Dict[int, NodeRecord],
        domain_node_id: int,
        is_code: bool,
    ) -> Dict[int, int]:
        nonlocal next_id
        id_map: Dict[int, int] = {}

        # Sort by (level, node_id) for stable assignment.
        for old_id, rec in sorted(src_nodes.items(), key=lambda kv: (kv[1].level, kv[0])):
            if rec.level == 0:
                # Skip local root; we replace it with shared root + domain nodes.
                continue

            new_level = rec.level + 1  # shift deeper by one level
            new_id = next_id
            next_id += 1

            # Determine parent in joint space.
            if rec.parent_id is None or rec.parent_id == 0:
                new_parent = domain_node_id
            else:
                if rec.parent_id not in id_map:
                    raise ValueError(
                        f"Parent node_id {rec.parent_id} for node {old_id} has not been remapped yet."
                    )
                new_parent = id_map[rec.parent_id]

            # Merge attributes and tag source.
            attrs = dict(rec.attributes or {})
            attrs.setdefault("source", "code" if is_code else "paper")

            joint_nodes[new_id] = NodeRecord(
                node_id=new_id,
                level=new_level,
                type=rec.type,
                parent_id=new_parent,
                name=rec.name,
                attributes=attrs,
            )

            id_map[old_id] = new_id
            level_counts[new_level] = level_counts.get(new_level, 0) + 1

        return id_map

    code_id_map = _remap_domain(src_nodes=code_nodes, domain_node_id=code_domain_id, is_code=True)
    paper_id_map = _remap_domain(
        src_nodes=paper_nodes, domain_node_id=paper_domain_id, is_code=False
    )

    return joint_nodes, code_id_map, paper_id_map, level_counts

def _rewrite_spans(
    spans_path: Path,
    *,
    domain_node_id: int,
    id_map: Dict[int, int],
) -> List[Dict[str, object]]:
    """
    Rewrite node_path for spans from a single domain into the joint ID space.
    """
    out: List[Dict[str, object]] = []
    root_id = 0

    for rec in _iter_spans(spans_path):
        old_path = rec.get("node_path")
        if not isinstance(old_path, list) or not old_path:
            # Leave record untouched if it lacks a path.
            out.append(rec)
            continue

        # Old paths are expected to be [root_id, ... local node ids ...].
        # We drop the old root and map each remaining id via id_map.
        new_path: List[int] = [root_id, domain_node_id]
        for old_id in old_path[1:]:
            new_id = id_map.get(int(old_id))
            if new_id is None:
                # If we don't know this node, skip it; better a shorter path
                # than a broken one.
                continue
            new_path.append(new_id)

        # Ensure at least [root, domain] is present.
        if len(new_path) < 2:
            new_path = [root_id, domain_node_id]

        rec = dict(rec)
        rec["node_path"] = new_path
        out.append(rec)

    return out

def _write_nodes_jsonl(nodes: Dict[int, NodeRecord], out_path: Path) -> None:
    with out_path.open("w", encoding="utf-8") as f:
        for nid, rec in sorted(nodes.items(), key=lambda kv: kv[0]):
            f.write(
                json.dumps(
                    {
                        "node_id": rec.node_id,
                        "level": rec.level,
                        "type": rec.type,
                        "parent_id": rec.parent_id,
                        "name": rec.name,
                        "attributes": rec.attributes,
                    }
                )
                + "\n"
            )

def _write_level_sizes(level_counts: Dict[int, int], out_path: Path) -> None:
    # Exclude root (level 0) from level_sizes; TOLBERT heads start at level 1.
    level_sizes = {level: count for level, count in level_counts.items() if level > 0}
    with out_path.open("w", encoding="utf-8") as f:
        json.dump({"level_sizes": level_sizes}, f, indent=2)

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(
        description="Build a joint code+paper Tree-of-Life from per-domain spans and nodes.",
    )
    ap.add_argument("--code_spans", type=str, required=True, help="Code spans JSONL file.")
    ap.add_argument("--code_nodes", type=str, required=True, help="Code nodes JSONL file.")
    ap.add_argument("--paper_spans", type=str, required=True, help="Paper spans JSONL file.")
    ap.add_argument("--paper_nodes", type=str, required=True, help="Paper nodes JSONL file.")

    ap.add_argument(
        "--out_code_spans",
        type=str,
        required=True,
        help="Output path for code spans JSONL with joint node_path.",
    )
    ap.add_argument(
        "--out_paper_spans",
        type=str,
        required=True,
        help="Output path for paper spans JSONL with joint node_path.",
    )
    ap.add_argument(
        "--out_nodes",
        type=str,
        required=True,
        help="Output path for joint nodes JSONL.",
    )
    ap.add_argument(
        "--out_level_sizes",
        type=str,
        required=True,
        help="Output path for joint level_sizes JSON helper JSON.",
    )

    ap.add_argument(
        "--code_domain_name",
        type=str,
        default="Code",
        help="Name for the code domain node at level 1.",
    )
    ap.add_argument(
        "--paper_domain_name",
        type=str,
        default="Papers",
        help="Name for the paper domain node at level 1.",
    )

    return ap.parse_args()

def main() -> None:
    args = parse_args()

    code_nodes_path = Path(args.code_nodes)
    paper_nodes_path = Path(args.paper_nodes)
    code_spans_path = Path(args.code_spans)
    paper_spans_path = Path(args.paper_spans)

    if not code_nodes_path.is_file():
        raise FileNotFoundError(f"code_nodes does not exist or is not a file: {code_nodes_path}")
    if not paper_nodes_path.is_file():
        raise FileNotFoundError(
            f"paper_nodes does not exist or is not a file: {paper_nodes_path}"
        )
    if not code_spans_path.is_file():
        raise FileNotFoundError(f"code_spans does not exist or is not a file: {code_spans_path}")
    if not paper_spans_path.is_file():
        raise FileNotFoundError(
            f"paper_spans does not exist or is not a file: {paper_spans_path}"
        )

    code_nodes = _load_nodes(code_nodes_path)
    paper_nodes = _load_nodes(paper_nodes_path)

    joint_nodes, code_id_map, paper_id_map, level_counts = _build_joint_ontology(
        code_nodes=code_nodes,
        paper_nodes=paper_nodes,
        code_domain_name=args.code_domain_name,
        paper_domain_name=args.paper_domain_name,
    )

    # Rewrite spans into the joint ID space.
    # Domain node IDs are always 1 (code) and 2 (paper) per _build_joint_ontology.
    code_domain_id = 1
    paper_domain_id = 2
    code_spans_joint = _rewrite_spans(
        code_spans_path,
        domain_node_id=code_domain_id,
        id_map=code_id_map,
    )
    paper_spans_joint = _rewrite_spans(
        paper_spans_path,
        domain_node_id=paper_domain_id,
        id_map=paper_id_map,
    )

    out_code_spans_path = Path(args.out_code_spans)
    out_paper_spans_path = Path(args.out_paper_spans)
    out_nodes_path = Path(args.out_nodes)
    out_level_sizes_path = Path(args.out_level_sizes)

    out_code_spans_path.parent.mkdir(parents=True, exist_ok=True)
    out_paper_spans_path.parent.mkdir(parents=True, exist_ok=True)
    out_nodes_path.parent.mkdir(parents=True, exist_ok=True)
    out_level_sizes_path.parent.mkdir(parents=True, exist_ok=True)

    _write_spans(out_code_spans_path, code_spans_joint)
    _write_spans(out_paper_spans_path, paper_spans_joint)
    _write_nodes_jsonl(joint_nodes, out_nodes_path)
    _write_level_sizes(level_counts, out_level_sizes_path)

if __name__ == "__main__":
    main()


NodeRecord

        --out_code_spans data/joint/code_spans_train.jsonl \
        --out_paper_spans data/joint/paper_spans_train.jsonl \
        --out_nodes data/joint/nodes.jsonl \
        --out_level_sizes data/joint/level_sizes.json

You can then train TOLBERT on the union of `out_code_spans` and
`out_paper_spans` (e.g., via the `spans_files` list in your config) with
`level_sizes` taken from `out_level_sizes`.
"""

from __future__ import annotations

import argparse
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Tuple

@dataclass
class NodeRecord:
    node_id: int
    level: int
    type: str
    parent_id: int | None
    name: str
    attributes: Dict[str, object]

def _load_nodes(path: Path) -> Dict[int, NodeRecord]:
    nodes: Dict[int, NodeRecord] = {}
    with path.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            obj = json.loads(line)
            nid = int(obj["node_id"])
            level = int(obj.get("level", 0))
            node_type = str(obj.get("type", "node"))
            parent_id = obj.get("parent_id")
            if parent_id is not None:
                parent_id = int(parent_id)
            name = str(obj.get("name", f"node_{nid}"))
            attributes = obj.get("attributes") or {}
            nodes[nid] = NodeRecord(
                node_id=nid,


from __future__ import annotations

import argparse
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Iterable, List, Tuple

@dataclass
class NodeRecord:
    node_id: int
    level: int
    type: str
    parent_id: int | None
    name: str
    attributes: Dict[str, object]

def _load_nodes(path: Path) -> Dict[int, NodeRecord]:
    nodes: Dict[int, NodeRecord] = {}
    with path.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            obj = json.loads(line)
            nid = int(obj["node_id"])
            level = int(obj.get("level", 0))
            node_type = str(obj.get("type", "node"))
            parent_id = obj.get("parent_id")
            if parent_id is not None:
                parent_id = int(parent_id)
            name = str(obj.get("name", f"node_{nid}"))
            attributes = obj.get("attributes") or {}
            nodes[nid] = NodeRecord(
                node_id=nid,
                level=level,
                type=node_type,
                parent_id=parent_id,
                name=name,
                attributes=attributes,
            )
    return nodes

def _iter_spans(path: Path) -> Iterable[Dict[str, object]]:
    with path.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            yield json.loads(line)

def _write_spans(path: Path, records: Iterable[Dict[str, object]]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

def _build_joint_ontology(
    *,
    code_nodes: Dict[int, NodeRecord],


            obj = json.loads(line)
            nid = int(obj["node_id"])
            level = int(obj.get("level", 0))
            node_type = str(obj.get("type", "node"))
            parent_id = obj.get("parent_id")
            if parent_id is not None:
                parent_id = int(parent_id)
            name = str(obj.get("name", f"node_{nid}"))
            attributes = obj.get("attributes") or {}
            nodes[nid] = NodeRecord(
                node_id=nid,
                level=level,
                type=node_type,
                parent_id=parent_id,
                name=name,
                attributes=attributes,
            )
    return nodes

def _iter_spans(path: Path) -> Iterable[Dict[str, object]]:
    with path.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            yield json.loads(line)

def _write_spans(path: Path, records: Iterable[Dict[str, object]]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

def _build_joint_ontology(
    *,
    code_nodes: Dict[int, NodeRecord],
    paper_nodes: Dict[int, NodeRecord],
    code_domain_name: str = "Code",
    paper_domain_name: str = "Papers",
) -> Tuple[
    Dict[int, NodeRecord],
    Dict[int, int],
    Dict[int, int],
    Dict[int, int],
]:


            nodes[nid] = NodeRecord(
                node_id=nid,
                level=level,
                type=node_type,
                parent_id=parent_id,
                name=name,
                attributes=attributes,
            )
    return nodes

def _iter_spans(path: Path) -> Iterable[Dict[str, object]]:
    with path.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            yield json.loads(line)

def _write_spans(path: Path, records: Iterable[Dict[str, object]]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

def _build_joint_ontology(
    *,
    code_nodes: Dict[int, NodeRecord],
    paper_nodes: Dict[int, NodeRecord],
    code_domain_name: str = "Code",
    paper_domain_name: str = "Papers",
) -> Tuple[
    Dict[int, NodeRecord],
    Dict[int, int],
    Dict[int, int],
    Dict[int, int],
]:
    """
    Construct a joint ontology and return:

    - joint_nodes: dict[new_id -> NodeRecord]
    - code_id_map: mapping old_code_node_id -> new_global_node_id
    - paper_id_map: mapping old_paper_node_id -> new_global_node_id


                attributes=attributes,
            )
    return nodes

def _iter_spans(path: Path) -> Iterable[Dict[str, object]]:
    with path.open("r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if not line:
                continue
            yield json.loads(line)

def _write_spans(path: Path, records: Iterable[Dict[str, object]]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

def _build_joint_ontology(
    *,
    code_nodes: Dict[int, NodeRecord],
    paper_nodes: Dict[int, NodeRecord],
    code_domain_name: str = "Code",
    paper_domain_name: str = "Papers",
) -> Tuple[
    Dict[int, NodeRecord],
    Dict[int, int],
    Dict[int, int],
    Dict[int, int],
]:
    """
    Construct a joint ontology and return:

    - joint_nodes: dict[new_id -> NodeRecord]
    - code_id_map: mapping old_code_node_id -> new_global_node_id
    - paper_id_map: mapping old_paper_node_id -> new_global_node_id
    - level_counts: mapping level -> number of nodes at that level (excluding root)

    Strategy:
      - Create a fresh root (id 0).
      - Create two domain nodes at level 1: Code, Papers.
      - For each original ontology:
          * Drop its root node (level 0).
          * Shift all other nodes one level deeper (level' = level + 1).
          * Rewire parents so that:
              - old parent == 0  → new parent = corresponding domain node id
              - otherwise        → new parent = mapped parent id
    """
    joint_nodes: Dict[int, NodeRecord] = {}
    level_counts: Dict[int, int] = {}

    # Root (level 0)
    root_id = 0
    joint_nodes[root_id] = NodeRecord(
        node_id=root_id,
        level=0,
        type="root",
        parent_id=None,
        name="Root",
        attributes={},
    )

    next_id = 1

    # Domain nodes at level 1
    code_domain_id = next_id
    next_id += 1
    paper_domain_id = next_id
    next_id += 1

    joint_nodes[code_domain_id] = NodeRecord(
        node_id=code_domain_id,
        level=1,
        type="domain",
        parent_id=root_id,
        name=code_domain_name,
        attributes={"source": "code"},
    )
    joint_nodes[paper_domain_id] = NodeRecord(
        node_id=paper_domain_id,
        level=1,
        type="domain",
        parent_id=root_id,
        name=paper_domain_name,
        attributes={"source": "paper"},
    )
    level_counts[1] = 2

    code_id_map: Dict[int, int] = {}
    paper_id_map: Dict[int, int] = {}

    def _remap_domain(
        *,
        src_nodes: Dict[int, NodeRecord],
        domain_node_id: int,
        is_code: bool,
    ) -> Dict[int, int]:
        nonlocal next_id
        id_map: Dict[int, int] = {}

        # Sort by (level, node_id) for stable assignment.
        for old_id, rec in sorted(src_nodes.items(), key=lambda kv: (kv[1].level, kv[0])):
            if rec.level == 0:
                # Skip local root; we replace it with shared root + domain nodes.
                continue

            new_level = rec.level + 1  # shift deeper by one level
            new_id = next_id
            next_id += 1

            # Determine parent in joint space.
            if rec.parent_id is None or rec.parent_id == 0:
                new_parent = domain_node_id
            else:
                if rec.parent_id not in id_map:
                    raise ValueError(
                        f"Parent node_id {rec.parent_id} for node {old_id} has not been remapped yet."
                    )
                new_parent = id_map[rec.parent_id]

            # Merge attributes and tag source.
            attrs = dict(rec.attributes or {})
            attrs.setdefault("source", "code" if is_code else "paper")

            joint_nodes[new_id] = NodeRecord(
                node_id=new_id,
                level=new_level,
                type=rec.type,
                parent_id=new_parent,
                name=rec.name,
                attributes=attrs,
            )

            id_map[old_id] = new_id
            level_counts[new_level] = level_counts.get(new_level, 0) + 1

        return id_map

    code_id_map = _remap_domain(src_nodes=code_nodes, domain_node_id=code_domain_id, is_code=True)
    paper_id_map = _remap_domain(
        src_nodes=paper_nodes, domain_node_id=paper_domain_id, is_code=False
    )

    return joint_nodes, code_id_map, paper_id_map, level_counts

def _rewrite_spans(
    spans_path: Path,
    *,
    domain_node_id: int,
    id_map: Dict[int, int],
) -> List[Dict[str, object]]:
    """
    Rewrite node_path for spans from a single domain into the joint ID space.
    """
    out: List[Dict[str, object]] = []
    root_id = 0

    for rec in _iter_spans(spans_path):
        old_path = rec.get("node_path")
        if not isinstance(old_path, list) or not old_path:
            # Leave record untouched if it lacks a path.
            out.append(rec)
            continue


                level=new_level,
                type=rec.type,
                parent_id=new_parent,
                name=rec.name,
                attributes=attrs,
            )

            id_map[old_id] = new_id
            level_counts[new_level] = level_counts.get(new_level, 0) + 1

        return id_map

    code_id_map = _remap_domain(src_nodes=code_nodes, domain_node_id=code_domain_id, is_code=True)
    paper_id_map = _remap_domain(
        src_nodes=paper_nodes, domain_node_id=paper_domain_id, is_code=False
    )

    return joint_nodes, code_id_map, paper_id_map, level_counts

def _rewrite_spans(
    spans_path: Path,
    *,
    domain_node_id: int,
    id_map: Dict[int, int],
) -> List[Dict[str, object]]:
    """
    Rewrite node_path for spans from a single domain into the joint ID space.
    """
    out: List[Dict[str, object]] = []
    root_id = 0

    for rec in _iter_spans(spans_path):
        old_path = rec.get("node_path")
        if not isinstance(old_path, list) or not old_path:
            # Leave record untouched if it lacks a path.
            out.append(rec)
            continue

        # Old paths are expected to be [root_id, ... local node ids ...].
        # We drop the old root and map each remaining id via id_map.
        new_path: List[int] = [root_id, domain_node_id]
        for old_id in old_path[1:]:
            new_id = id_map.get(int(old_id))
            if new_id is None:
                # If we don't know this node, skip it; better a shorter path
                # than a broken one.
                continue
            new_path.append(new_id)

        # Ensure at least [root, domain] is present.
        if len(new_path) < 2:
            new_path = [root_id, domain_node_id]

        rec = dict(rec)
        rec["node_path"] = new_path
        out.append(rec)

    return out

def _write_nodes_jsonl(nodes: Dict[int, NodeRecord], out_path: Path) -> None:
    with out_path.open("w", encoding="utf-8") as f:
        for nid, rec in sorted(nodes.items(), key=lambda kv: kv[0]):
            f.write(
                json.dumps(
                    {
                        "node_id": rec.node_id,
                        "level": rec.level,
                        "type": rec.type,
                        "parent_id": rec.parent_id,
                        "name": rec.name,
                        "attributes": rec.attributes,
                    }
                )
                + "\n"
            )


        new_path: List[int] = [root_id, domain_node_id]
        for old_id in old_path[1:]:
            new_id = id_map.get(int(old_id))
            if new_id is None:
                # If we don't know this node, skip it; better a shorter path
                # than a broken one.
                continue
            new_path.append(new_id)

        # Ensure at least [root, domain] is present.
        if len(new_path) < 2:
            new_path = [root_id, domain_node_id]

        rec = dict(rec)
        rec["node_path"] = new_path
        out.append(rec)

    return out

def _write_nodes_jsonl(nodes: Dict[int, NodeRecord], out_path: Path) -> None:
    with out_path.open("w", encoding="utf-8") as f:
        for nid, rec in sorted(nodes.items(), key=lambda kv: kv[0]):
            f.write(
                json.dumps(
                    {
                        "node_id": rec.node_id,
                        "level": rec.level,
                        "type": rec.type,
                        "parent_id": rec.parent_id,
                        "name": rec.name,
                        "attributes": rec.attributes,
                    }
                )
                + "\n"
            )

def _write_level_sizes(level_counts: Dict[int, int], out_path: Path) -> None:
    # Exclude root (level 0) from level_sizes; TOLBERT heads start at level 1.
    level_sizes = {level: count for level, count in level_counts.items() if level > 0}
    with out_path.open("w", encoding="utf-8") as f:
        json.dump({"level_sizes": level_sizes}, f, indent=2)

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(
        description="Build a joint code+paper Tree-of-Life from per-domain spans and nodes.",
    )
    ap.add_argument("--code_spans", type=str, required=True, help="Code spans JSONL file.")
    ap.add_argument("--code_nodes", type=str, required=True, help="Code nodes JSONL file.")
    ap.add_argument("--paper_spans", type=str, required=True, help="Paper spans JSONL file.")
    ap.add_argument("--paper_nodes", type=str, required=True, help="Paper nodes JSONL file.")

    ap.add_argument(
        "--out_code_spans",


def _write_nodes_jsonl(nodes: Dict[int, NodeRecord], out_path: Path) -> None:
    with out_path.open("w", encoding="utf-8") as f:
        for nid, rec in sorted(nodes.items(), key=lambda kv: kv[0]):
            f.write(
                json.dumps(
                    {
                        "node_id": rec.node_id,
                        "level": rec.level,
                        "type": rec.type,
                        "parent_id": rec.parent_id,
                        "name": rec.name,
                        "attributes": rec.attributes,
                    }
                )
                + "\n"
            )

def _write_level_sizes(level_counts: Dict[int, int], out_path: Path) -> None:
    # Exclude root (level 0) from level_sizes; TOLBERT heads start at level 1.
    level_sizes = {level: count for level, count in level_counts.items() if level > 0}
    with out_path.open("w", encoding="utf-8") as f:
        json.dump({"level_sizes": level_sizes}, f, indent=2)

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(
        description="Build a joint code+paper Tree-of-Life from per-domain spans and nodes.",
    )
    ap.add_argument("--code_spans", type=str, required=True, help="Code spans JSONL file.")
    ap.add_argument("--code_nodes", type=str, required=True, help="Code nodes JSONL file.")
    ap.add_argument("--paper_spans", type=str, required=True, help="Paper spans JSONL file.")
    ap.add_argument("--paper_nodes", type=str, required=True, help="Paper nodes JSONL file.")

    ap.add_argument(
        "--out_code_spans",
        type=str,
        required=True,
        help="Output path for code spans JSONL with joint node_path.",
    )
    ap.add_argument(
        "--out_paper_spans",
        type=str,