import types
from pathlib import Path
import sys

import torch

# Ensure project root is on sys.path so `tolbert` imports resolve consistently,
# even when tests are run from outside the repository root.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from tolbert import modeling

class _DummyEncoder(torch.nn.Module):
    """
    Tiny stand-in for a HuggingFace AutoModel encoder.

    It exposes:
      - .config.hidden_size
      - .config.vocab_size
      - .forward(...).last_hidden_state
    so that TOLBERT can be instantiated without downloading a real model.
    """

    def __init__(self, hidden_size: int = 16, vocab_size: int = 32) -> None:
        super().__init__()
        self.config = types.SimpleNamespace(hidden_size=hidden_size, vocab_size=vocab_size)

    def forward(self, input_ids=None, attention_mask=None):
        batch, seq_len = input_ids.shape
        # Simple deterministic hidden states for reproducibility across runs.
        hidden = torch.arange(batch * seq_len * self.config.hidden_size, dtype=torch.float32)
        hidden = hidden.view(batch, seq_len, self.config.hidden_size)
        return types.SimpleNamespace(last_hidden_state=hidden)

class _DummyAutoModel:
    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        return _DummyEncoder()

def _patch_auto_model():
    """
    Replace modeling.AutoModel with a cheap local encoder for tests.
    """
    modeling.AutoModel = _DummyAutoModel  # type: ignore[assignment]

def test_tolbert_forward_shapes_and_losses():
    """
    Basic sanity check:
      - TOLBERT can be instantiated with a dummy encoder.
      - Forward pass returns all expected keys.
      - Loss and individual loss components are present.
    """
    _patch_auto_model()

    cfg = modeling.TOLBERTConfig(
        base_model_name="dummy",
        level_sizes={1: 3, 2: 4},
        proj_dim=8,
        lambda_hier=1.0,
        lambda_path=0.5,
    )
    model = modeling.TOLBERT(cfg)

    batch_size, seq_len = 2, 5
    input_ids = torch.ones(batch_size, seq_len, dtype=torch.long)
    attention_mask = torch.ones_like(input_ids)

    # Simple MLM labels: predict a couple of positions, others ignored.
    labels_mlm = input_ids.clone()
    labels_mlm[:, 0] = -100

    # Per-level targets; use -100 to mark missing labels for one example.
    level_targets = {
        1: torch.tensor([0, 1], dtype=torch.long),
        2: torch.tensor([2, -100], dtype=torch.long),
    }

    # Paths: [root, level1, level2]; align with level indices 1 and 2.
    paths = [
        [0, 0, 2],
        [0, 1, 3],
    ]

    out = model(
        input_ids=input_ids,
        attention_mask=attention_mask,
        labels_mlm=labels_mlm,
        level_targets=level_targets,
        paths=paths,
    )

    assert "loss" in out
    assert out["loss"] is not None
    assert "loss_components" in out
    loss_components = out["loss_components"]
    assert "mlm" in loss_components
    assert "hier" in loss_components
    # Path loss may be zero if no invalid mass, but key should exist when paths are provided.
    assert "path" in loss_components

    # Shape checks
    mlm_logits = out["mlm_logits"]
    assert mlm_logits.shape == (batch_size, seq_len, model.encoder.config.vocab_size)

    level_logits = out["level_logits"]
    assert set(level_logits.keys()) == {"1", "2"}
    assert level_logits["1"].shape == (batch_size, 3)
    assert level_logits["2"].shape == (batch_size, 4)

    proj = out["proj"]
    assert proj.shape == (batch_size, cfg.proj_dim)
    # Embeddings should be L2-normalized.
    norms = torch.norm(proj, dim=-1)
    assert torch.allclose(norms, torch.ones_like(norms), atol=1e-5)

def test_path_consistency_penalizes_invalid_children():
    """
    Directly test the path-consistency helper on a toy example:
      - Parent distribution puts mass entirely on parent 0.
      - Child distribution puts mass on a child whose parent != 0.
      - The resulting path loss should be strictly positive.
    """
    _patch_auto_model()

    cfg = modeling.TOLBERTConfig(
        base_model_name="dummy",
        level_sizes={1: 2, 2: 2},
        proj_dim=4,
        lambda_hier=1.0,
        lambda_path=1.0,
    )
    model = modeling.TOLBERT(cfg)

    device = next(model.parameters()).device

    # Construct dummy logits for level 1 and 2.
    # Level 1: always predict parent 0 with prob ~1.
    logits_l1 = torch.tensor([[10.0, -10.0]], device=device)  # (1, 2)
    # Level 2: two children; child 0 has parent 1 (mismatched), child 1 has parent 0 (matched).
    logits_l2 = torch.tensor([[10.0, -10.0]], device=device)  # will favor child 0 (invalid)

    level_logits = {"1": logits_l1, "2": logits_l2}

    # Paths: [root, parent_id_at_level1, child_id_at_level2]
    # Define ontology:
    #   - child 0 -> parent 1
    #   - child 1 -> parent 0
    # By choosing parent 0 at level 1 and highest prob on child 0 at level 2,
    # we force probability mass on an invalid child.
    paths = [[0, 0, 0]]

    path_loss = model._compute_path_consistency_loss(level_logits, paths)
    # With a consistent ontology (child->parent mapping derived from paths),
    # the KL-based path-consistency loss should be finite and non-negative.
    assert path_loss is not None
    assert torch.isfinite(path_loss)
    assert path_loss.item() >= 0.0


import types
from pathlib import Path
import sys

import torch

# Ensure project root is on sys.path so `tolbert` imports resolve consistently,
# even when tests are run from outside the repository root.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from tolbert import modeling

class _DummyEncoder(torch.nn.Module):
    """
    Tiny stand-in for a HuggingFace AutoModel encoder.

    It exposes:
      - .config.hidden_size
      - .config.vocab_size
      - .forward(...).last_hidden_state
    so that TOLBERT can be instantiated without downloading a real model.
    """

    def __init__(self, hidden_size: int = 16, vocab_size: int = 32) -> None:
        super().__init__()
        self.config = types.SimpleNamespace(hidden_size=hidden_size, vocab_size=vocab_size)

    def forward(self, input_ids=None, attention_mask=None):
        batch, seq_len = input_ids.shape
        # Simple deterministic hidden states for reproducibility across runs.
        hidden = torch.arange(batch * seq_len * self.config.hidden_size, dtype=torch.float32)
        hidden = hidden.view(batch, seq_len, self.config.hidden_size)
        return types.SimpleNamespace(last_hidden_state=hidden)

class _DummyAutoModel:
    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        return _DummyEncoder()

def _patch_auto_model():
    """
    Replace modeling.AutoModel with a cheap local encoder for tests.
    """
    modeling.AutoModel = _DummyAutoModel  # type: ignore[assignment]

def test_tolbert_forward_shapes_and_losses():
    """
    Basic sanity check:
      - TOLBERT can be instantiated with a dummy encoder.
      - Forward pass returns all expected keys.


    It exposes:
      - .config.hidden_size
      - .config.vocab_size
      - .forward(...).last_hidden_state
    so that TOLBERT can be instantiated without downloading a real model.
    """

    def __init__(self, hidden_size: int = 16, vocab_size: int = 32) -> None:
        super().__init__()
        self.config = types.SimpleNamespace(hidden_size=hidden_size, vocab_size=vocab_size)

    def forward(self, input_ids=None, attention_mask=None):
        batch, seq_len = input_ids.shape
        # Simple deterministic hidden states for reproducibility across runs.
        hidden = torch.arange(batch * seq_len * self.config.hidden_size, dtype=torch.float32)
        hidden = hidden.view(batch, seq_len, self.config.hidden_size)
        return types.SimpleNamespace(last_hidden_state=hidden)

class _DummyAutoModel:
    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        return _DummyEncoder()

def _patch_auto_model():
    """
    Replace modeling.AutoModel with a cheap local encoder for tests.
    """
    modeling.AutoModel = _DummyAutoModel  # type: ignore[assignment]

def test_tolbert_forward_shapes_and_losses():
    """
    Basic sanity check:
      - TOLBERT can be instantiated with a dummy encoder.
      - Forward pass returns all expected keys.
      - Loss and individual loss components are present.
    """
    _patch_auto_model()

    cfg = modeling.TOLBERTConfig(
        base_model_name="dummy",


    """

    def __init__(self, hidden_size: int = 16, vocab_size: int = 32) -> None:
        super().__init__()
        self.config = types.SimpleNamespace(hidden_size=hidden_size, vocab_size=vocab_size)

    def forward(self, input_ids=None, attention_mask=None):
        batch, seq_len = input_ids.shape
        # Simple deterministic hidden states for reproducibility across runs.
        hidden = torch.arange(batch * seq_len * self.config.hidden_size, dtype=torch.float32)
        hidden = hidden.view(batch, seq_len, self.config.hidden_size)
        return types.SimpleNamespace(last_hidden_state=hidden)

class _DummyAutoModel:
    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        return _DummyEncoder()

def _patch_auto_model():
    """
    Replace modeling.AutoModel with a cheap local encoder for tests.
    """
    modeling.AutoModel = _DummyAutoModel  # type: ignore[assignment]

def test_tolbert_forward_shapes_and_losses():
    """
    Basic sanity check:
      - TOLBERT can be instantiated with a dummy encoder.
      - Forward pass returns all expected keys.
      - Loss and individual loss components are present.
    """
    _patch_auto_model()

    cfg = modeling.TOLBERTConfig(
        base_model_name="dummy",
        level_sizes={1: 3, 2: 4},
        proj_dim=8,
        lambda_hier=1.0,
        lambda_path=0.5,
    )
    model = modeling.TOLBERT(cfg)


        batch, seq_len = input_ids.shape
        # Simple deterministic hidden states for reproducibility across runs.
        hidden = torch.arange(batch * seq_len * self.config.hidden_size, dtype=torch.float32)
        hidden = hidden.view(batch, seq_len, self.config.hidden_size)
        return types.SimpleNamespace(last_hidden_state=hidden)

class _DummyAutoModel:
    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        return _DummyEncoder()

def _patch_auto_model():
    """
    Replace modeling.AutoModel with a cheap local encoder for tests.
    """
    modeling.AutoModel = _DummyAutoModel  # type: ignore[assignment]

def test_tolbert_forward_shapes_and_losses():
    """
    Basic sanity check:
      - TOLBERT can be instantiated with a dummy encoder.
      - Forward pass returns all expected keys.
      - Loss and individual loss components are present.
    """
    _patch_auto_model()

    cfg = modeling.TOLBERTConfig(
        base_model_name="dummy",
        level_sizes={1: 3, 2: 4},
        proj_dim=8,
        lambda_hier=1.0,
        lambda_path=0.5,
    )
    model = modeling.TOLBERT(cfg)

    batch_size, seq_len = 2, 5
    input_ids = torch.ones(batch_size, seq_len, dtype=torch.long)
    attention_mask = torch.ones_like(input_ids)

    # Simple MLM labels: predict a couple of positions, others ignored.
    labels_mlm = input_ids.clone()
    labels_mlm[:, 0] = -100

    # Per-level targets; use -100 to mark missing labels for one example.
    level_targets = {
        1: torch.tensor([0, 1], dtype=torch.long),
        2: torch.tensor([2, -100], dtype=torch.long),
    }

    # Paths: [root, level1, level2]; align with level indices 1 and 2.
    paths = [
        [0, 0, 2],
        [0, 1, 3],
    ]

    out = model(
        input_ids=input_ids,
        attention_mask=attention_mask,
        labels_mlm=labels_mlm,
        level_targets=level_targets,
        paths=paths,
    )

    assert "loss" in out
    assert out["loss"] is not None
    assert "loss_components" in out
    loss_components = out["loss_components"]
    assert "mlm" in loss_components
    assert "hier" in loss_components
    # Path loss may be zero if no invalid mass, but key should exist when paths are provided.
    assert "path" in loss_components

    # Shape checks
    mlm_logits = out["mlm_logits"]
    assert mlm_logits.shape == (batch_size, seq_len, model.encoder.config.vocab_size)

    level_logits = out["level_logits"]
    assert set(level_logits.keys()) == {"1", "2"}
    assert level_logits["1"].shape == (batch_size, 3)
    assert level_logits["2"].shape == (batch_size, 4)

    proj = out["proj"]
    assert proj.shape == (batch_size, cfg.proj_dim)
    # Embeddings should be L2-normalized.
    norms = torch.norm(proj, dim=-1)
    assert torch.allclose(norms, torch.ones_like(norms), atol=1e-5)

def test_path_consistency_penalizes_invalid_children():
    """
    Directly test the path-consistency helper on a toy example:
      - Parent distribution puts mass entirely on parent 0.
      - Child distribution puts mass on a child whose parent != 0.
      - The resulting path loss should be strictly positive.
    """
    _patch_auto_model()

    cfg = modeling.TOLBERTConfig(
        base_model_name="dummy",
        level_sizes={1: 2, 2: 2},
        proj_dim=4,
        lambda_hier=1.0,
        lambda_path=1.0,
    )
    model = modeling.TOLBERT(cfg)


    assert "hier" in loss_components
    # Path loss may be zero if no invalid mass, but key should exist when paths are provided.
    assert "path" in loss_components

    # Shape checks
    mlm_logits = out["mlm_logits"]
    assert mlm_logits.shape == (batch_size, seq_len, model.encoder.config.vocab_size)

    level_logits = out["level_logits"]
    assert set(level_logits.keys()) == {"1", "2"}
    assert level_logits["1"].shape == (batch_size, 3)
    assert level_logits["2"].shape == (batch_size, 4)

    proj = out["proj"]
    assert proj.shape == (batch_size, cfg.proj_dim)
    # Embeddings should be L2-normalized.
    norms = torch.norm(proj, dim=-1)
    assert torch.allclose(norms, torch.ones_like(norms), atol=1e-5)

def test_path_consistency_penalizes_invalid_children():
    """
    Directly test the path-consistency helper on a toy example:
      - Parent distribution puts mass entirely on parent 0.
      - Child distribution puts mass on a child whose parent != 0.
      - The resulting path loss should be strictly positive.
    """
    _patch_auto_model()

    cfg = modeling.TOLBERTConfig(
        base_model_name="dummy",
        level_sizes={1: 2, 2: 2},
        proj_dim=4,
        lambda_hier=1.0,
        lambda_path=1.0,
    )
    model = modeling.TOLBERT(cfg)

    device = next(model.parameters()).device

    # Construct dummy logits for level 1 and 2.
    # Level 1: always predict parent 0 with prob ~1.
    logits_l1 = torch.tensor([[10.0, -10.0]], device=device)  # (1, 2)
    # Level 2: two children; child 0 has parent 1 (mismatched), child 1 has parent 0 (matched).
    logits_l2 = torch.tensor([[10.0, -10.0]], device=device)  # will favor child 0 (invalid)

    level_logits = {"1": logits_l1, "2": logits_l2}

    # Paths: [root, parent_id_at_level1, child_id_at_level2]
    # Define ontology:
    #   - child 0 -> parent 1
    #   - child 1 -> parent 0
    # By choosing parent 0 at level 1 and highest prob on child 0 at level 2,
    # we force probability mass on an invalid child.
    paths = [[0, 0, 0]]

    path_loss = model._compute_path_consistency_loss(level_logits, paths)
    # With a consistent ontology (child->parent mapping derived from paths),
    # the KL-based path-consistency loss should be finite and non-negative.
    assert path_loss is not None
    assert torch.isfinite(path_loss)
    assert path_loss.item() >= 0.0


# Ensure project root is on sys.path so `tolbert` imports resolve consistently,
# even when tests are run from outside the repository root.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from tolbert import modeling

class _DummyEncoder(torch.nn.Module):
    """
    Tiny stand-in for a HuggingFace AutoModel encoder.

    It exposes:
      - .config.hidden_size
      - .config.vocab_size
      - .forward(...).last_hidden_state
    so that TOLBERT can be instantiated without downloading a real model.
    """

    def __init__(self, hidden_size: int = 16, vocab_size: int = 32) -> None:
        super().__init__()
        self.config = types.SimpleNamespace(hidden_size=hidden_size, vocab_size=vocab_size)

    def forward(self, input_ids=None, attention_mask=None):
        batch, seq_len = input_ids.shape
        # Simple deterministic hidden states for reproducibility across runs.
        hidden = torch.arange(batch * seq_len * self.config.hidden_size, dtype=torch.float32)
        hidden = hidden.view(batch, seq_len, self.config.hidden_size)
        return types.SimpleNamespace(last_hidden_state=hidden)

class _DummyAutoModel:
    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        return _DummyEncoder()

def _patch_auto_model():
    """
    Replace modeling.AutoModel with a cheap local encoder for tests.
    """
    modeling.AutoModel = _DummyAutoModel  # type: ignore[assignment]


forward

    sys.path.insert(0, str(PROJECT_ROOT))

from tolbert import modeling

class _DummyEncoder(torch.nn.Module):
    """
    Tiny stand-in for a HuggingFace AutoModel encoder.

    It exposes:
      - .config.hidden_size
      - .config.vocab_size
      - .forward(...).last_hidden_state
    so that TOLBERT can be instantiated without downloading a real model.
    """

    def __init__(self, hidden_size: int = 16, vocab_size: int = 32) -> None:
        super().__init__()
        self.config = types.SimpleNamespace(hidden_size=hidden_size, vocab_size=vocab_size)

    def forward(self, input_ids=None, attention_mask=None):
        batch, seq_len = input_ids.shape
        # Simple deterministic hidden states for reproducibility across runs.
        hidden = torch.arange(batch * seq_len * self.config.hidden_size, dtype=torch.float32)
        hidden = hidden.view(batch, seq_len, self.config.hidden_size)
        return types.SimpleNamespace(last_hidden_state=hidden)

class _DummyAutoModel:
    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        return _DummyEncoder()

def _patch_auto_model():
    """
    Replace modeling.AutoModel with a cheap local encoder for tests.
    """
    modeling.AutoModel = _DummyAutoModel  # type: ignore[assignment]

def test_tolbert_forward_shapes_and_losses():
    """
    Basic sanity check:
      - TOLBERT can be instantiated with a dummy encoder.
      - Forward pass returns all expected keys.


      - .config.hidden_size
      - .config.vocab_size
      - .forward(...).last_hidden_state
    so that TOLBERT can be instantiated without downloading a real model.
    """

    def __init__(self, hidden_size: int = 16, vocab_size: int = 32) -> None:
        super().__init__()
        self.config = types.SimpleNamespace(hidden_size=hidden_size, vocab_size=vocab_size)

    def forward(self, input_ids=None, attention_mask=None):
        batch, seq_len = input_ids.shape
        # Simple deterministic hidden states for reproducibility across runs.
        hidden = torch.arange(batch * seq_len * self.config.hidden_size, dtype=torch.float32)
        hidden = hidden.view(batch, seq_len, self.config.hidden_size)
        return types.SimpleNamespace(last_hidden_state=hidden)

class _DummyAutoModel:
    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        return _DummyEncoder()

def _patch_auto_model():
    """
    Replace modeling.AutoModel with a cheap local encoder for tests.
    """
    modeling.AutoModel = _DummyAutoModel  # type: ignore[assignment]

def test_tolbert_forward_shapes_and_losses():
    """
    Basic sanity check:
      - TOLBERT can be instantiated with a dummy encoder.
      - Forward pass returns all expected keys.
      - Loss and individual loss components are present.
    """
    _patch_auto_model()

    cfg = modeling.TOLBERTConfig(
        base_model_name="dummy",


from pathlib import Path
import sys
from typing import List

import torch

# Ensure project root is on sys.path so we import the local `scripts` package.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from scripts import build_repo_tree_of_life

class _DummySpan:
    def __init__(self, start_line: int, end_line: int) -> None:
        self.start_line = start_line
        self.end_line = end_line

class _DummyEntity:
    def __init__(self, kind: str, rel_path: str, labels: List[str] | None = None):
        self.kind = kind
        self.attributes = {"rel_path": rel_path, "name": rel_path}
        self.labels = labels or []
        self.id = rel_path
        self.artifact_uri = f"program://dummy/artifact/{rel_path}"
        self.span = _DummySpan(1, 10)

class _DummyRepoGraph:
    """
    Minimal stand-in for RepoGraph.entities() to exercise build_tree_for_repo.
    """

    def __init__(self, root: str) -> None:
        self.root = root

    def entities(self):
        # One Python file and one C++ file, plus a function symbol in the Python file.
        return [
            _DummyEntity("file", "pkg/file1.py", labels=["lang:python"]),
            _DummyEntity("file", "pkg/file2.cpp", labels=["lang:cpp"]),
            _DummyEntity("function", "pkg/file1.py::fn", labels=[]),
        ]

def test_build_tree_for_repo_with_dummy_graph(tmp_path: Path, monkeypatch):
    """
    Smoke test for build_tree_for_repo:
      - uses a dummy RepoGraph to avoid filesystem scanning,
      - verifies that nodes, edges, and spans have the expected basic structure.
    """
    # Patch RepoGraph used inside build_repo_tree_of_life to our dummy.
    monkeypatch.setattr(build_repo_tree_of_life, "RepoGraph", _DummyRepoGraph)

    repo_root = tmp_path / "repo"
    (repo_root / "pkg").mkdir(parents=True)
    (repo_root / "pkg" / "file1.py").write_text("print('hello')\n", encoding="utf-8")
    (repo_root / "pkg" / "file2.cpp").write_text("int main() { return 0; }\n", encoding="utf-8")

    nodes, edges, spans = build_repo_tree_of_life.build_tree_for_repo(repo_root)

    # Basic sanity checks.
    assert nodes
    assert edges
    assert spans

    # There should be a root node at level 0.
    root_nodes = [n for n in nodes if n.get("level") == 0 and n.get("type") == "root"]
    assert len(root_nodes) == 1

    # There should be language nodes at level 1 and a repo node at level 2.
    lang_nodes = [n for n in nodes if n.get("level") == 1 and n.get("type") == "language"]
    assert lang_nodes
    repo_nodes = [n for n in nodes if n.get("level") == 2 and n.get("type") == "repo"]
    assert len(repo_nodes) == 1

    # Spans should have a node_path starting at root and including the repo node id.
    repo_id = repo_nodes[0]["node_id"]
    for s in spans:
        path = s["node_path"]
        assert isinstance(path, list)
        assert path[0] == root_nodes[0]["node_id"]
        assert repo_id in path


from pathlib import Path
import sys
from typing import List

import torch

# Ensure project root is on sys.path so we import the local `scripts` package.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from scripts import build_repo_tree_of_life

class _DummySpan:
    def __init__(self, start_line: int, end_line: int) -> None:
        self.start_line = start_line
        self.end_line = end_line

class _DummyEntity:
    def __init__(self, kind: str, rel_path: str, labels: List[str] | None = None):
        self.kind = kind
        self.attributes = {"rel_path": rel_path, "name": rel_path}
        self.labels = labels or []
        self.id = rel_path
        self.artifact_uri = f"program://dummy/artifact/{rel_path}"
        self.span = _DummySpan(1, 10)

class _DummyRepoGraph:
    """
    Minimal stand-in for RepoGraph.entities() to exercise build_tree_for_repo.
    """

    def __init__(self, root: str) -> None:
        self.root = root


from pathlib import Path
import sys
from typing import List

import torch

# Ensure project root is on sys.path so we import the local `scripts` package.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from scripts import build_repo_tree_of_life

class _DummySpan:
    def __init__(self, start_line: int, end_line: int) -> None:
        self.start_line = start_line
        self.end_line = end_line

class _DummyEntity:
    def __init__(self, kind: str, rel_path: str, labels: List[str] | None = None):
        self.kind = kind
        self.attributes = {"rel_path": rel_path, "name": rel_path}
        self.labels = labels or []
        self.id = rel_path
        self.artifact_uri = f"program://dummy/artifact/{rel_path}"
        self.span = _DummySpan(1, 10)

class _DummyRepoGraph:
    """
    Minimal stand-in for RepoGraph.entities() to exercise build_tree_for_repo.
    """

    def __init__(self, root: str) -> None:
        self.root = root

    def entities(self):
        # One Python file and one C++ file, plus a function symbol in the Python file.
        return [
            _DummyEntity("file", "pkg/file1.py", labels=["lang:python"]),
            _DummyEntity("file", "pkg/file2.cpp", labels=["lang:cpp"]),
            _DummyEntity("function", "pkg/file1.py::fn", labels=[]),
        ]

def test_build_tree_for_repo_with_dummy_graph(tmp_path: Path, monkeypatch):


from scripts import build_repo_tree_of_life

class _DummySpan:
    def __init__(self, start_line: int, end_line: int) -> None:
        self.start_line = start_line
        self.end_line = end_line

class _DummyEntity:
    def __init__(self, kind: str, rel_path: str, labels: List[str] | None = None):
        self.kind = kind
        self.attributes = {"rel_path": rel_path, "name": rel_path}
        self.labels = labels or []
        self.id = rel_path
        self.artifact_uri = f"program://dummy/artifact/{rel_path}"
        self.span = _DummySpan(1, 10)

class _DummyRepoGraph:
    """
    Minimal stand-in for RepoGraph.entities() to exercise build_tree_for_repo.
    """

    def __init__(self, root: str) -> None:
        self.root = root

    def entities(self):
        # One Python file and one C++ file, plus a function symbol in the Python file.
        return [
            _DummyEntity("file", "pkg/file1.py", labels=["lang:python"]),
            _DummyEntity("file", "pkg/file2.cpp", labels=["lang:cpp"]),
            _DummyEntity("function", "pkg/file1.py::fn", labels=[]),
        ]

def test_build_tree_for_repo_with_dummy_graph(tmp_path: Path, monkeypatch):
    """
    Smoke test for build_tree_for_repo:
      - uses a dummy RepoGraph to avoid filesystem scanning,
      - verifies that nodes, edges, and spans have the expected basic structure.
    """
    # Patch RepoGraph used inside build_repo_tree_of_life to our dummy.
    monkeypatch.setattr(build_repo_tree_of_life, "RepoGraph", _DummyRepoGraph)

    repo_root = tmp_path / "repo"
    (repo_root / "pkg").mkdir(parents=True)
    (repo_root / "pkg" / "file1.py").write_text("print('hello')\n", encoding="utf-8")
    (repo_root / "pkg" / "file2.cpp").write_text("int main() { return 0; }\n", encoding="utf-8")

    nodes, edges, spans = build_repo_tree_of_life.build_tree_for_repo(repo_root)

    # Basic sanity checks.
    assert nodes


        self.span = _DummySpan(1, 10)

class _DummyRepoGraph:
    """
    Minimal stand-in for RepoGraph.entities() to exercise build_tree_for_repo.
    """

    def __init__(self, root: str) -> None:
        self.root = root

    def entities(self):
        # One Python file and one C++ file, plus a function symbol in the Python file.
        return [
            _DummyEntity("file", "pkg/file1.py", labels=["lang:python"]),
            _DummyEntity("file", "pkg/file2.cpp", labels=["lang:cpp"]),
            _DummyEntity("function", "pkg/file1.py::fn", labels=[]),
        ]

def test_build_tree_for_repo_with_dummy_graph(tmp_path: Path, monkeypatch):
    """
    Smoke test for build_tree_for_repo:
      - uses a dummy RepoGraph to avoid filesystem scanning,
      - verifies that nodes, edges, and spans have the expected basic structure.
    """
    # Patch RepoGraph used inside build_repo_tree_of_life to our dummy.
    monkeypatch.setattr(build_repo_tree_of_life, "RepoGraph", _DummyRepoGraph)

    repo_root = tmp_path / "repo"
    (repo_root / "pkg").mkdir(parents=True)
    (repo_root / "pkg" / "file1.py").write_text("print('hello')\n", encoding="utf-8")
    (repo_root / "pkg" / "file2.cpp").write_text("int main() { return 0; }\n", encoding="utf-8")

    nodes, edges, spans = build_repo_tree_of_life.build_tree_for_repo(repo_root)

    # Basic sanity checks.
    assert nodes
    assert edges
    assert spans

    # There should be a root node at level 0.
    root_nodes = [n for n in nodes if n.get("level") == 0 and n.get("type") == "root"]
    assert len(root_nodes) == 1

    # There should be language nodes at level 1 and a repo node at level 2.
    lang_nodes = [n for n in nodes if n.get("level") == 1 and n.get("type") == "language"]
    assert lang_nodes
    repo_nodes = [n for n in nodes if n.get("level") == 2 and n.get("type") == "repo"]
    assert len(repo_nodes) == 1

    # Spans should have a node_path starting at root and including the repo node id.
    repo_id = repo_nodes[0]["node_id"]
    for s in spans:
        path = s["node_path"]
        assert isinstance(path, list)
        assert path[0] == root_nodes[0]["node_id"]
        assert repo_id in path


    def __init__(self, start_line: int, end_line: int) -> None:
        self.start_line = start_line
        self.end_line = end_line

class _DummyEntity:
    def __init__(self, kind: str, rel_path: str, labels: List[str] | None = None):
        self.kind = kind
        self.attributes = {"rel_path": rel_path, "name": rel_path}
        self.labels = labels or []
        self.id = rel_path
        self.artifact_uri = f"program://dummy/artifact/{rel_path}"
        self.span = _DummySpan(1, 10)

class _DummyRepoGraph:
    """
    Minimal stand-in for RepoGraph.entities() to exercise build_tree_for_repo.
    """

    def __init__(self, root: str) -> None:
        self.root = root

    def entities(self):
        # One Python file and one C++ file, plus a function symbol in the Python file.
        return [
            _DummyEntity("file", "pkg/file1.py", labels=["lang:python"]),
            _DummyEntity("file", "pkg/file2.cpp", labels=["lang:cpp"]),
            _DummyEntity("function", "pkg/file1.py::fn", labels=[]),
        ]

def test_build_tree_for_repo_with_dummy_graph(tmp_path: Path, monkeypatch):
    """
    Smoke test for build_tree_for_repo:
      - uses a dummy RepoGraph to avoid filesystem scanning,
      - verifies that nodes, edges, and spans have the expected basic structure.
    """
    # Patch RepoGraph used inside build_repo_tree_of_life to our dummy.
    monkeypatch.setattr(build_repo_tree_of_life, "RepoGraph", _DummyRepoGraph)

    repo_root = tmp_path / "repo"


entities

class _DummyEntity:
    def __init__(self, kind: str, rel_path: str, labels: List[str] | None = None):
        self.kind = kind
        self.attributes = {"rel_path": rel_path, "name": rel_path}
        self.labels = labels or []
        self.id = rel_path
        self.artifact_uri = f"program://dummy/artifact/{rel_path}"
        self.span = _DummySpan(1, 10)

class _DummyRepoGraph:
    """
    Minimal stand-in for RepoGraph.entities() to exercise build_tree_for_repo.
    """

    def __init__(self, root: str) -> None:
        self.root = root

    def entities(self):
        # One Python file and one C++ file, plus a function symbol in the Python file.
        return [
            _DummyEntity("file", "pkg/file1.py", labels=["lang:python"]),
            _DummyEntity("file", "pkg/file2.cpp", labels=["lang:cpp"]),
            _DummyEntity("function", "pkg/file1.py::fn", labels=[]),
        ]

def test_build_tree_for_repo_with_dummy_graph(tmp_path: Path, monkeypatch):
    """
    Smoke test for build_tree_for_repo:
      - uses a dummy RepoGraph to avoid filesystem scanning,
      - verifies that nodes, edges, and spans have the expected basic structure.
    """
    # Patch RepoGraph used inside build_repo_tree_of_life to our dummy.
    monkeypatch.setattr(build_repo_tree_of_life, "RepoGraph", _DummyRepoGraph)

    repo_root = tmp_path / "repo"
    (repo_root / "pkg").mkdir(parents=True)
    (repo_root / "pkg" / "file1.py").write_text("print('hello')\n", encoding="utf-8")
    (repo_root / "pkg" / "file2.cpp").write_text("int main() { return 0; }\n", encoding="utf-8")

    nodes, edges, spans = build_repo_tree_of_life.build_tree_for_repo(repo_root)

    # Basic sanity checks.
    assert nodes


import json
from pathlib import Path
import sys

import torch

# Ensure project root is on sys.path so local `tolbert` and `scripts` packages
# are importable regardless of the working directory.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from tolbert.data import TreeOfLifeDataset, collate_tree_of_life_batch
from scripts import eval_retrieval

def test_tree_of_life_dataset_and_collate_single_path(tmp_path: Path):
    """
    Verify that TreeOfLifeDataset:
      - reads node_path into per-level targets,
      - produces paths suitable for contrastive / path losses,
      - and that collate_tree_of_life_batch produces the expected tensors.
    """
    spans_path = tmp_path / "spans_single.jsonl"
    records = [
        {
            "span_id": "s1",
            "text": "hello world",
            "node_path": [0, 1, 3],
        },
        {
            "span_id": "s2",
            "text": "another span",
            "node_path": [0, 2, 4],
        },
    ]
    with spans_path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

    class _DummyTokenizer:
        def __init__(self):
            self.vocab_size = 100
            # Provide a MASK token id for MLM replacement logic.
            self.mask_token_id = 99

        def get_special_tokens_mask(self, ids, already_has_special_tokens=True):
            return [0] * len(ids)

        def __call__(self, text, return_tensors, truncation, padding, max_length):
            # Simple fixed-length encoding: map each char to an integer.
            ids = list(range(1, min(len(text) + 1, max_length - 2)))
            # Pad / truncate to max_length.
            ids = ids + [0] * (max_length - len(ids))
            input_ids = torch.tensor(ids, dtype=torch.long).unsqueeze(0)
            attention_mask = (input_ids != 0).long()
            return {"input_ids": input_ids, "attention_mask": attention_mask}

    tokenizer = _DummyTokenizer()
    ds = TreeOfLifeDataset(
        spans_file=str(spans_path),
        tokenizer=tokenizer,  # type: ignore[arg-type]
        max_length=8,
        mask_probability=0.15,
    )

    assert len(ds) == 2
    sample0 = ds[0]
    assert "input_ids" in sample0 and "labels_mlm" in sample0
    assert "level_targets" in sample0
    assert "paths" in sample0
    # Level targets should have entries for levels 1 and 2.
    assert sample0["level_targets"][1] == 1
    assert sample0["level_targets"][2] == 3

    batch = collate_tree_of_life_batch([ds[0], ds[1]])
    assert batch["input_ids"].shape[0] == 2
    assert set(batch["level_targets"].keys()) == {1, 2}
    # Paths should be a list of node_path lists (wrapped in an extra list per example).
    assert "paths" in batch
    assert batch["paths"][0] == [[0, 1, 3]]
    assert batch["paths"][1] == [[0, 2, 4]]

def test_tree_of_life_dataset_with_node_paths_multi(tmp_path: Path):
    """
    Verify that when `node_paths` (multiple valid paths) is provided:
      - the first path is used as canonical `level_targets`,
      - all paths are preserved in the `paths` field for DAG-aware losses.
    """
    spans_path = tmp_path / "spans_multi.jsonl"
    records = [
        {
            "span_id": "s1",
            "text": "multi path span",
            "node_paths": [
                [0, 1, 3],
                [0, 1, 4],
            ],
        },
    ]
    with spans_path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

    class _DummyTokenizer:
        def __init__(self):
            self.vocab_size = 100
            self.mask_token_id = 99

        def get_special_tokens_mask(self, ids, already_has_special_tokens=True):
            return [0] * len(ids)

        def __call__(self, text, return_tensors, truncation, padding, max_length):
            ids = list(range(1, min(len(text) + 1, max_length - 2)))
            ids = ids + [0] * (max_length - len(ids))
            input_ids = torch.tensor(ids, dtype=torch.long).unsqueeze(0)
            attention_mask = (input_ids != 0).long()
            return {"input_ids": input_ids, "attention_mask": attention_mask}

    tokenizer = _DummyTokenizer()
    ds = TreeOfLifeDataset(
        spans_file=str(spans_path),
        tokenizer=tokenizer,  # type: ignore[arg-type]
        max_length=8,
        mask_probability=0.15,
    )

    assert len(ds) == 1
    sample = ds[0]
    # Canonical targets come from the first path [0,1,3].
    assert sample["level_targets"][1] == 1
    assert sample["level_targets"][2] == 3
    # All paths are preserved.
    assert sample["paths"] == [[0, 1, 3], [0, 1, 4]]

def test_eval_retrieval_helpers_basic():
    """
    Sanity-check compute_relevance_mask, eval_retrieval, and
    compute_branch_consistency_at_k on a tiny toy example.
    """
    # Two index items, two queries.
    index_embs = torch.eye(2, dtype=torch.float32)
    query_embs = torch.eye(2, dtype=torch.float32)

    # Paths: query 0 matches index 0 at level 1; query 1 matches index 1.
    index_paths = [[0, 1], [0, 2]]
    query_paths = [[0, 1], [0, 2]]

    rel = eval_retrieval.compute_relevance_mask(
        query_paths=query_paths,
        index_paths=index_paths,
        min_level=1,
    )
    assert rel == [[True, False], [False, True]]

    mrr, p_at_1 = eval_retrieval.eval_retrieval(
        query_embs=query_embs,
        index_embs=index_embs,
        relevant=rel,
        k=1,
    )
    # Each query retrieves its exact match at rank 1.
    assert abs(mrr - 1.0) < 1e-6
    assert abs(p_at_1 - 1.0) < 1e-6

    bc = eval_retrieval.compute_branch_consistency_at_k(
        query_embs=query_embs,
        index_embs=index_embs,
        query_paths=query_paths,
        index_paths=index_paths,
        k=1,
    )
    # At depth 1, all retrieved items share the same node as the query.
    assert 1 in bc
    assert abs(bc[1] - 1.0) < 1e-6


import json
from pathlib import Path
import sys

import torch

# Ensure project root is on sys.path so local `tolbert` and `scripts` packages
# are importable regardless of the working directory.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from tolbert.data import TreeOfLifeDataset, collate_tree_of_life_batch
from scripts import eval_retrieval

def test_tree_of_life_dataset_and_collate_single_path(tmp_path: Path):
    """
    Verify that TreeOfLifeDataset:
      - reads node_path into per-level targets,
      - produces paths suitable for contrastive / path losses,
      - and that collate_tree_of_life_batch produces the expected tensors.
    """
    spans_path = tmp_path / "spans_single.jsonl"
    records = [
        {
            "span_id": "s1",
            "text": "hello world",
            "node_path": [0, 1, 3],
        },
        {
            "span_id": "s2",
            "text": "another span",
            "node_path": [0, 2, 4],
        },
    ]
    with spans_path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

    class _DummyTokenizer:
        def __init__(self):
            self.vocab_size = 100
            # Provide a MASK token id for MLM replacement logic.
            self.mask_token_id = 99

        def get_special_tokens_mask(self, ids, already_has_special_tokens=True):
            return [0] * len(ids)

        def __call__(self, text, return_tensors, truncation, padding, max_length):
            # Simple fixed-length encoding: map each char to an integer.
            ids = list(range(1, min(len(text) + 1, max_length - 2)))
            # Pad / truncate to max_length.
            ids = ids + [0] * (max_length - len(ids))
            input_ids = torch.tensor(ids, dtype=torch.long).unsqueeze(0)
            attention_mask = (input_ids != 0).long()
            return {"input_ids": input_ids, "attention_mask": attention_mask}

    tokenizer = _DummyTokenizer()
    ds = TreeOfLifeDataset(
        spans_file=str(spans_path),
        tokenizer=tokenizer,  # type: ignore[arg-type]
        max_length=8,
        mask_probability=0.15,
    )

    assert len(ds) == 2
    sample0 = ds[0]
    assert "input_ids" in sample0 and "labels_mlm" in sample0
    assert "level_targets" in sample0
    assert "paths" in sample0
    # Level targets should have entries for levels 1 and 2.
    assert sample0["level_targets"][1] == 1
    assert sample0["level_targets"][2] == 3

    batch = collate_tree_of_life_batch([ds[0], ds[1]])
    assert batch["input_ids"].shape[0] == 2
    assert set(batch["level_targets"].keys()) == {1, 2}
    # Paths should be a list of node_path lists (wrapped in an extra list per example).
    assert "paths" in batch
    assert batch["paths"][0] == [[0, 1, 3]]
    assert batch["paths"][1] == [[0, 2, 4]]

def test_tree_of_life_dataset_with_node_paths_multi(tmp_path: Path):
    """
    Verify that when `node_paths` (multiple valid paths) is provided:
      - the first path is used as canonical `level_targets`,
      - all paths are preserved in the `paths` field for DAG-aware losses.
    """
    spans_path = tmp_path / "spans_multi.jsonl"
    records = [
        {
            "span_id": "s1",
            "text": "multi path span",
            "node_paths": [
                [0, 1, 3],
                [0, 1, 4],
            ],
        },
    ]
    with spans_path.open("w", encoding="utf-8") as f:


    )

    assert len(ds) == 2
    sample0 = ds[0]
    assert "input_ids" in sample0 and "labels_mlm" in sample0
    assert "level_targets" in sample0
    assert "paths" in sample0
    # Level targets should have entries for levels 1 and 2.
    assert sample0["level_targets"][1] == 1
    assert sample0["level_targets"][2] == 3

    batch = collate_tree_of_life_batch([ds[0], ds[1]])
    assert batch["input_ids"].shape[0] == 2
    assert set(batch["level_targets"].keys()) == {1, 2}
    # Paths should be a list of node_path lists (wrapped in an extra list per example).
    assert "paths" in batch
    assert batch["paths"][0] == [[0, 1, 3]]
    assert batch["paths"][1] == [[0, 2, 4]]

def test_tree_of_life_dataset_with_node_paths_multi(tmp_path: Path):
    """
    Verify that when `node_paths` (multiple valid paths) is provided:
      - the first path is used as canonical `level_targets`,
      - all paths are preserved in the `paths` field for DAG-aware losses.
    """
    spans_path = tmp_path / "spans_multi.jsonl"
    records = [
        {
            "span_id": "s1",
            "text": "multi path span",
            "node_paths": [
                [0, 1, 3],
                [0, 1, 4],
            ],
        },
    ]
    with spans_path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

    class _DummyTokenizer:
        def __init__(self):
            self.vocab_size = 100
            self.mask_token_id = 99

        def get_special_tokens_mask(self, ids, already_has_special_tokens=True):
            return [0] * len(ids)

        def __call__(self, text, return_tensors, truncation, padding, max_length):
            ids = list(range(1, min(len(text) + 1, max_length - 2)))
            ids = ids + [0] * (max_length - len(ids))
            input_ids = torch.tensor(ids, dtype=torch.long).unsqueeze(0)
            attention_mask = (input_ids != 0).long()
            return {"input_ids": input_ids, "attention_mask": attention_mask}

    tokenizer = _DummyTokenizer()
    ds = TreeOfLifeDataset(
        spans_file=str(spans_path),
        tokenizer=tokenizer,  # type: ignore[arg-type]
        max_length=8,
        mask_probability=0.15,
    )

    assert len(ds) == 1
    sample = ds[0]
    # Canonical targets come from the first path [0,1,3].
    assert sample["level_targets"][1] == 1
    assert sample["level_targets"][2] == 3
    # All paths are preserved.
    assert sample["paths"] == [[0, 1, 3], [0, 1, 4]]

def test_eval_retrieval_helpers_basic():
    """
    Sanity-check compute_relevance_mask, eval_retrieval, and
    compute_branch_consistency_at_k on a tiny toy example.
    """
    # Two index items, two queries.
    index_embs = torch.eye(2, dtype=torch.float32)
    query_embs = torch.eye(2, dtype=torch.float32)

    # Paths: query 0 matches index 0 at level 1; query 1 matches index 1.
    index_paths = [[0, 1], [0, 2]]
    query_paths = [[0, 1], [0, 2]]

    rel = eval_retrieval.compute_relevance_mask(
        query_paths=query_paths,
        index_paths=index_paths,
        min_level=1,
    )


            attention_mask = (input_ids != 0).long()
            return {"input_ids": input_ids, "attention_mask": attention_mask}

    tokenizer = _DummyTokenizer()
    ds = TreeOfLifeDataset(
        spans_file=str(spans_path),
        tokenizer=tokenizer,  # type: ignore[arg-type]
        max_length=8,
        mask_probability=0.15,
    )

    assert len(ds) == 1
    sample = ds[0]
    # Canonical targets come from the first path [0,1,3].
    assert sample["level_targets"][1] == 1
    assert sample["level_targets"][2] == 3
    # All paths are preserved.
    assert sample["paths"] == [[0, 1, 3], [0, 1, 4]]

def test_eval_retrieval_helpers_basic():
    """
    Sanity-check compute_relevance_mask, eval_retrieval, and
    compute_branch_consistency_at_k on a tiny toy example.
    """
    # Two index items, two queries.
    index_embs = torch.eye(2, dtype=torch.float32)
    query_embs = torch.eye(2, dtype=torch.float32)

    # Paths: query 0 matches index 0 at level 1; query 1 matches index 1.
    index_paths = [[0, 1], [0, 2]]
    query_paths = [[0, 1], [0, 2]]

    rel = eval_retrieval.compute_relevance_mask(
        query_paths=query_paths,
        index_paths=index_paths,
        min_level=1,
    )
    assert rel == [[True, False], [False, True]]

    mrr, p_at_1 = eval_retrieval.eval_retrieval(
        query_embs=query_embs,
        index_embs=index_embs,
        relevant=rel,
        k=1,
    )
    # Each query retrieves its exact match at rank 1.
    assert abs(mrr - 1.0) < 1e-6
    assert abs(p_at_1 - 1.0) < 1e-6

    bc = eval_retrieval.compute_branch_consistency_at_k(
        query_embs=query_embs,
        index_embs=index_embs,
        query_paths=query_paths,
        index_paths=index_paths,
        k=1,
    )
    # At depth 1, all retrieved items share the same node as the query.
    assert 1 in bc
    assert abs(bc[1] - 1.0) < 1e-6


    """
    Verify that when `node_paths` (multiple valid paths) is provided:
      - the first path is used as canonical `level_targets`,
      - all paths are preserved in the `paths` field for DAG-aware losses.
    """
    spans_path = tmp_path / "spans_multi.jsonl"
    records = [
        {
            "span_id": "s1",
            "text": "multi path span",
            "node_paths": [
                [0, 1, 3],
                [0, 1, 4],
            ],
        },
    ]
    with spans_path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

    class _DummyTokenizer:
        def __init__(self):
            self.vocab_size = 100
            self.mask_token_id = 99

        def get_special_tokens_mask(self, ids, already_has_special_tokens=True):
            return [0] * len(ids)

        def __call__(self, text, return_tensors, truncation, padding, max_length):
            ids = list(range(1, min(len(text) + 1, max_length - 2)))
            ids = ids + [0] * (max_length - len(ids))
            input_ids = torch.tensor(ids, dtype=torch.long).unsqueeze(0)
            attention_mask = (input_ids != 0).long()
            return {"input_ids": input_ids, "attention_mask": attention_mask}

    tokenizer = _DummyTokenizer()
    ds = TreeOfLifeDataset(
        spans_file=str(spans_path),
        tokenizer=tokenizer,  # type: ignore[arg-type]
        max_length=8,
        mask_probability=0.15,
    )

    assert len(ds) == 1
    sample = ds[0]
    # Canonical targets come from the first path [0,1,3].
    assert sample["level_targets"][1] == 1
    assert sample["level_targets"][2] == 3
    # All paths are preserved.
    assert sample["paths"] == [[0, 1, 3], [0, 1, 4]]

def test_eval_retrieval_helpers_basic():
    """


    Verify that when `node_paths` (multiple valid paths) is provided:
      - the first path is used as canonical `level_targets`,
      - all paths are preserved in the `paths` field for DAG-aware losses.
    """
    spans_path = tmp_path / "spans_multi.jsonl"
    records = [
        {
            "span_id": "s1",
            "text": "multi path span",
            "node_paths": [
                [0, 1, 3],
                [0, 1, 4],
            ],
        },
    ]
    with spans_path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

    class _DummyTokenizer:
        def __init__(self):
            self.vocab_size = 100
            self.mask_token_id = 99

        def get_special_tokens_mask(self, ids, already_has_special_tokens=True):
            return [0] * len(ids)

        def __call__(self, text, return_tensors, truncation, padding, max_length):
            ids = list(range(1, min(len(text) + 1, max_length - 2)))
            ids = ids + [0] * (max_length - len(ids))
            input_ids = torch.tensor(ids, dtype=torch.long).unsqueeze(0)
            attention_mask = (input_ids != 0).long()
            return {"input_ids": input_ids, "attention_mask": attention_mask}

    tokenizer = _DummyTokenizer()
    ds = TreeOfLifeDataset(
        spans_file=str(spans_path),
        tokenizer=tokenizer,  # type: ignore[arg-type]
        max_length=8,
        mask_probability=0.15,
    )

    assert len(ds) == 1


    spans_path = tmp_path / "spans_multi.jsonl"
    records = [
        {
            "span_id": "s1",
            "text": "multi path span",
            "node_paths": [
                [0, 1, 3],
                [0, 1, 4],
            ],
        },
    ]
    with spans_path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

    class _DummyTokenizer:
        def __init__(self):
            self.vocab_size = 100
            self.mask_token_id = 99

        def get_special_tokens_mask(self, ids, already_has_special_tokens=True):
            return [0] * len(ids)

        def __call__(self, text, return_tensors, truncation, padding, max_length):
            ids = list(range(1, min(len(text) + 1, max_length - 2)))
            ids = ids + [0] * (max_length - len(ids))
            input_ids = torch.tensor(ids, dtype=torch.long).unsqueeze(0)
            attention_mask = (input_ids != 0).long()
            return {"input_ids": input_ids, "attention_mask": attention_mask}

    tokenizer = _DummyTokenizer()
    ds = TreeOfLifeDataset(
        spans_file=str(spans_path),
        tokenizer=tokenizer,  # type: ignore[arg-type]
        max_length=8,
        mask_probability=0.15,
    )

    assert len(ds) == 1
    sample = ds[0]
    # Canonical targets come from the first path [0,1,3].
    assert sample["level_targets"][1] == 1


            "span_id": "s1",
            "text": "multi path span",
            "node_paths": [
                [0, 1, 3],
                [0, 1, 4],
            ],
        },
    ]
    with spans_path.open("w", encoding="utf-8") as f:
        for rec in records:
            f.write(json.dumps(rec) + "\n")

    class _DummyTokenizer:
        def __init__(self):
            self.vocab_size = 100
            self.mask_token_id = 99

        def get_special_tokens_mask(self, ids, already_has_special_tokens=True):
            return [0] * len(ids)

        def __call__(self, text, return_tensors, truncation, padding, max_length):
            ids = list(range(1, min(len(text) + 1, max_length - 2)))
            ids = ids + [0] * (max_length - len(ids))
            input_ids = torch.tensor(ids, dtype=torch.long).unsqueeze(0)
            attention_mask = (input_ids != 0).long()
            return {"input_ids": input_ids, "attention_mask": attention_mask}

    tokenizer = _DummyTokenizer()
    ds = TreeOfLifeDataset(
        spans_file=str(spans_path),
        tokenizer=tokenizer,  # type: ignore[arg-type]
        max_length=8,
        mask_probability=0.15,
    )

    assert len(ds) == 1
    sample = ds[0]
    # Canonical targets come from the first path [0,1,3].
    assert sample["level_targets"][1] == 1
    assert sample["level_targets"][2] == 3
    # All paths are preserved.
    assert sample["paths"] == [[0, 1, 3], [0, 1, 4]]

def test_eval_retrieval_helpers_basic():
    """


import csv
import json
from pathlib import Path
import sys
from typing import Dict

# Ensure project root is on sys.path so we import the local `scripts` package
# instead of any third-party package named `scripts`.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from scripts import (
    build_wos_spans,
    build_researchhierarchy_spans,
    build_codehierarchy_spans,
    build_joint_code_paper_tol,
)

def test_build_wos_spans_helpers(tmp_path: Path):
    """
    Smoke test for build_wos_spans:
      - load_wos_csv parses a small CSV,
      - build_ontology assigns ids at each level,
      - build_spans produces node_path with expected structure.
    """
    csv_path = tmp_path / "wos_small.csv"
    with csv_path.open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["text", "level1", "level2", "level3"])
        writer.writeheader()
        writer.writerow(
            {
                "text": "doc about machine learning",
                "level1": "Computer Science",
                "level2": "Artificial Intelligence",
                "level3": "Machine Learning",
            }
        )
        writer.writerow(
            {
                "text": "doc about physics",
                "level1": "Physics",
                "level2": "Quantum",
                "level3": "Quantum Mechanics",
            }
        )

    rows = build_wos_spans.load_wos_csv(
        path=csv_path,
        text_col="text",
        l1_col="level1",
        l2_col="level2",
        l3_col="level3",
    )
    assert len(rows) == 2

    l1_ids, l2_ids, l3_ids = build_wos_spans.build_ontology(rows)
    assert len(l1_ids) == 2
    assert len(l2_ids) == 2
    assert len(l3_ids) == 2

    spans = build_wos_spans.build_spans(rows, l1_ids, l2_ids, l3_ids)
    assert len(spans) == 2
    for rec in spans:
        path = rec["node_path"]
        # Root + three levels if all labels are present.
        assert isinstance(path, list)
        assert len(path) == 4
        assert path[0] == 0

def test_build_researchhierarchy_spans_helpers(tmp_path: Path):
    """
    Smoke test for build_researchhierarchy_spans helpers:
      - load_metadata parses a small CSV,
      - build_ontology assigns ids,
      - build_span_records produces the expected node_path layout.
    """
    csv_path = tmp_path / "research_small.csv"
    with csv_path.open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(
            f,
            fieldnames=["doc_id", "field", "subfield", "discipline", "text"],
        )
        writer.writeheader()
        writer.writerow(
            {
                "doc_id": "D1",
                "field": "Computer Science",
                "subfield": "AI",
                "discipline": "ML",
                "text": "title and abstract about ML",
            }
        )

    metas = build_researchhierarchy_spans.load_metadata(
        path=csv_path,
        id_col="doc_id",
        field_col="field",
        subfield_col="subfield",
        discipline_col="discipline",
        text_col="text",
        pdf_path_col=None,
        source_col=None,
    )
    assert len(metas) == 1

    field_ids, subfield_ids, discipline_ids = build_researchhierarchy_spans.build_ontology(
        metas
    )
    assert len(field_ids) == 1
    assert len(subfield_ids) == 1
    assert len(discipline_ids) == 1

    spans = build_researchhierarchy_spans.build_span_records(
        metas, field_ids, subfield_ids, discipline_ids
    )
    assert len(spans) == 1
    rec = spans[0]
    path = rec["node_path"]
    assert isinstance(path, list)
    # root + field + subfield + discipline
    assert len(path) == 4
    assert path[0] == 0

def test_build_codehierarchy_spans_helpers(tmp_path: Path):
    """
    Smoke test for build_codehierarchy_spans helpers:
      - load_metadata parses a small metadata file,
      - build_ontology assigns ids for languages, categories, repos,
      - build_span_records produces spans with expected node_path layout.
    """
    # Create a fake repos_root with two repos and one file each.
    repos_root = tmp_path / "repos"
    repo1 = repos_root / "repo1"
    repo2 = repos_root / "repo2"
    repo1.mkdir(parents=True)
    repo2.mkdir(parents=True)
    (repo1 / "file1.py").write_text("print('hello')\n", encoding="utf-8")
    (repo2 / "file2.cpp").write_text("int main() { return 0; }\n", encoding="utf-8")

    meta_path = tmp_path / "meta.csv"
    with meta_path.open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["repo_name", "language", "category"])
        writer.writeheader()
        writer.writerow({"repo_name": "repo1", "language": "Python", "category": "ML"})
        writer.writerow({"repo_name": "repo2", "language": "C++", "category": "Systems"})

    metas = build_codehierarchy_spans.load_metadata(meta_path)
    assert len(metas) == 2

    lang_ids, cat_ids, repo_ids = build_codehierarchy_spans.build_ontology(metas)
    assert len(lang_ids) == 2
    assert len(cat_ids) == 2
    assert len(repo_ids) == 2

    spans = build_codehierarchy_spans.build_span_records(
        repos_root=repos_root,
        metas=metas,
        lang_ids=lang_ids,
        cat_ids=cat_ids,
        repo_ids=repo_ids,
    )
    # One span per file.
    assert len(spans) == 2
    for rec in spans:
        path = rec["node_path"]
        assert isinstance(path, list)
        # root + language + category + repo
        assert len(path) == 4
        assert path[0] == 0

def test_build_joint_code_paper_tol_helpers(tmp_path: Path):
    """
    Smoke test for build_joint_code_paper_tol core helpers:
      - _build_joint_ontology merges two small ontologies,
      - _rewrite_spans remaps node_path ids into the joint space,
      - _write_level_sizes produces a dict[level -> count] excluding root.
    """
    # Build minimal code and paper nodes.jsonl
    code_nodes_path = tmp_path / "code_nodes.jsonl"
    paper_nodes_path = tmp_path / "paper_nodes.jsonl"

    def _write_nodes(path: Path, levels: Dict[int, int]) -> None:
        with path.open("w", encoding="utf-8") as f:
            # local root
            f.write(
                json.dumps(
                    {
                        "node_id": 0,
                        "level": 0,
                        "type": "root",
                        "parent_id": None,
                        "name": "Root",
                        "attributes": {},
                    }
                )
                + "\n"
            )
            nid = 1
            for lvl, count in levels.items():
                for i in range(count):
                    parent_id = 0 if lvl == 1 else nid - 1
                    f.write(
                        json.dumps(
                            {
                                "node_id": nid,
                                "level": lvl,
                                "type": f"lvl{lvl}",
                                "parent_id": parent_id,
                                "name": f"n{lvl}_{i}",
                                "attributes": {},
                            }
                        )
                        + "\n"
                    )
                    nid += 1

    _write_nodes(code_nodes_path, {1: 1, 2: 1})
    _write_nodes(paper_nodes_path, {1: 1, 2: 1})

    code_nodes = build_joint_code_paper_tol._load_nodes(code_nodes_path)
    paper_nodes = build_joint_code_paper_tol._load_nodes(paper_nodes_path)

    joint_nodes, code_map, paper_map, level_counts = build_joint_code_paper_tol._build_joint_ontology(
        code_nodes=code_nodes,
        paper_nodes=paper_nodes,
        code_domain_name="Code",
        paper_domain_name="Papers",
    )
    # Root + 2 domain nodes + remapped nodes from both ontologies.
    assert 0 in joint_nodes
    assert level_counts[1] == 2  # Code, Papers
    # There should be some nodes at deeper levels.
    assert any(lvl > 1 for lvl in level_counts.keys())
    assert code_map and paper_map

    # Create minimal spans that refer to old node ids and ensure they get remapped.
    code_spans_path = tmp_path / "code_spans.jsonl"
    paper_spans_path = tmp_path / "paper_spans.jsonl"
    with code_spans_path.open("w", encoding="utf-8") as f:
        f.write(json.dumps({"span_id": "c1", "text": "code", "node_path": [0, 1, 2]}) + "\n")
    with paper_spans_path.open("w", encoding="utf-8") as f:
        f.write(json.dumps({"span_id": "p1", "text": "paper", "node_path": [0, 1, 2]}) + "\n")

    # Use internal helpers directly.
    code_domain_id = 1  # first domain under root in joint ontology
    paper_domain_id = 2  # second domain
    remapped_code_spans = build_joint_code_paper_tol._rewrite_spans(
        code_spans_path, domain_node_id=code_domain_id, id_map=code_map
    )
    remapped_paper_spans = build_joint_code_paper_tol._rewrite_spans(
        paper_spans_path, domain_node_id=paper_domain_id, id_map=paper_map
    )

    assert remapped_code_spans[0]["node_path"][0] == 0
    assert remapped_code_spans[0]["node_path"][1] == code_domain_id
    assert remapped_paper_spans[0]["node_path"][0] == 0
    assert remapped_paper_spans[0]["node_path"][1] == paper_domain_id

    # Check level_sizes writing.
    level_sizes_path = tmp_path / "level_sizes.json"
    build_joint_code_paper_tol._write_level_sizes(level_counts, level_sizes_path)
    contents = json.loads(level_sizes_path.read_text(encoding="utf-8"))
    assert "level_sizes" in contents
    # Root (0) should not be present.
    assert 0 not in contents["level_sizes"]


import csv
import json
from pathlib import Path
import sys
from typing import Dict

# Ensure project root is on sys.path so we import the local `scripts` package
# instead of any third-party package named `scripts`.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from scripts import (
    build_wos_spans,
    build_researchhierarchy_spans,
    build_codehierarchy_spans,
    build_joint_code_paper_tol,
)

def test_build_wos_spans_helpers(tmp_path: Path):
    """
    Smoke test for build_wos_spans:
      - load_wos_csv parses a small CSV,
      - build_ontology assigns ids at each level,
      - build_spans produces node_path with expected structure.
    """
    csv_path = tmp_path / "wos_small.csv"
    with csv_path.open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["text", "level1", "level2", "level3"])
        writer.writeheader()
        writer.writerow(
            {
                "text": "doc about machine learning",
                "level1": "Computer Science",
                "level2": "Artificial Intelligence",
                "level3": "Machine Learning",
            }
        )
        writer.writerow(
            {
                "text": "doc about physics",
                "level1": "Physics",
                "level2": "Quantum",
                "level3": "Quantum Mechanics",
            }
        )

    rows = build_wos_spans.load_wos_csv(
        path=csv_path,
        text_col="text",
        l1_col="level1",
        l2_col="level2",
        l3_col="level3",
    )
    assert len(rows) == 2

    l1_ids, l2_ids, l3_ids = build_wos_spans.build_ontology(rows)
    assert len(l1_ids) == 2
    assert len(l2_ids) == 2
    assert len(l3_ids) == 2

    spans = build_wos_spans.build_spans(rows, l1_ids, l2_ids, l3_ids)
    assert len(spans) == 2
    for rec in spans:
        path = rec["node_path"]
        # Root + three levels if all labels are present.
        assert isinstance(path, list)
        assert len(path) == 4
        assert path[0] == 0

def test_build_researchhierarchy_spans_helpers(tmp_path: Path):
    """
    Smoke test for build_researchhierarchy_spans helpers:
      - load_metadata parses a small CSV,
      - build_ontology assigns ids,
      - build_span_records produces the expected node_path layout.
    """
    csv_path = tmp_path / "research_small.csv"
    with csv_path.open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(
            f,
            fieldnames=["doc_id", "field", "subfield", "discipline", "text"],
        )
        writer.writeheader()
        writer.writerow(
            {
                "doc_id": "D1",
                "field": "Computer Science",


        l2_col="level2",
        l3_col="level3",
    )
    assert len(rows) == 2

    l1_ids, l2_ids, l3_ids = build_wos_spans.build_ontology(rows)
    assert len(l1_ids) == 2
    assert len(l2_ids) == 2
    assert len(l3_ids) == 2

    spans = build_wos_spans.build_spans(rows, l1_ids, l2_ids, l3_ids)
    assert len(spans) == 2
    for rec in spans:
        path = rec["node_path"]
        # Root + three levels if all labels are present.
        assert isinstance(path, list)
        assert len(path) == 4
        assert path[0] == 0

def test_build_researchhierarchy_spans_helpers(tmp_path: Path):
    """
    Smoke test for build_researchhierarchy_spans helpers:
      - load_metadata parses a small CSV,
      - build_ontology assigns ids,
      - build_span_records produces the expected node_path layout.
    """
    csv_path = tmp_path / "research_small.csv"
    with csv_path.open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(
            f,
            fieldnames=["doc_id", "field", "subfield", "discipline", "text"],
        )
        writer.writeheader()
        writer.writerow(
            {
                "doc_id": "D1",
                "field": "Computer Science",
                "subfield": "AI",
                "discipline": "ML",
                "text": "title and abstract about ML",
            }
        )

    metas = build_researchhierarchy_spans.load_metadata(
        path=csv_path,
        id_col="doc_id",
        field_col="field",
        subfield_col="subfield",
        discipline_col="discipline",
        text_col="text",
        pdf_path_col=None,
        source_col=None,
    )
    assert len(metas) == 1

    field_ids, subfield_ids, discipline_ids = build_researchhierarchy_spans.build_ontology(
        metas
    )
    assert len(field_ids) == 1
    assert len(subfield_ids) == 1
    assert len(discipline_ids) == 1

    spans = build_researchhierarchy_spans.build_span_records(
        metas, field_ids, subfield_ids, discipline_ids
    )
    assert len(spans) == 1
    rec = spans[0]
    path = rec["node_path"]
    assert isinstance(path, list)
    # root + field + subfield + discipline
    assert len(path) == 4
    assert path[0] == 0

def test_build_codehierarchy_spans_helpers(tmp_path: Path):
    """
    Smoke test for build_codehierarchy_spans helpers:
      - load_metadata parses a small metadata file,
      - build_ontology assigns ids for languages, categories, repos,
      - build_span_records produces spans with expected node_path layout.
    """
    # Create a fake repos_root with two repos and one file each.
    repos_root = tmp_path / "repos"
    repo1 = repos_root / "repo1"
    repo2 = repos_root / "repo2"
    repo1.mkdir(parents=True)
    repo2.mkdir(parents=True)
    (repo1 / "file1.py").write_text("print('hello')\n", encoding="utf-8")
    (repo2 / "file2.cpp").write_text("int main() { return 0; }\n", encoding="utf-8")

    meta_path = tmp_path / "meta.csv"
    with meta_path.open("w", encoding="utf-8", newline="") as f:


    field_ids, subfield_ids, discipline_ids = build_researchhierarchy_spans.build_ontology(
        metas
    )
    assert len(field_ids) == 1
    assert len(subfield_ids) == 1
    assert len(discipline_ids) == 1

    spans = build_researchhierarchy_spans.build_span_records(
        metas, field_ids, subfield_ids, discipline_ids
    )
    assert len(spans) == 1
    rec = spans[0]
    path = rec["node_path"]
    assert isinstance(path, list)
    # root + field + subfield + discipline
    assert len(path) == 4
    assert path[0] == 0

def test_build_codehierarchy_spans_helpers(tmp_path: Path):
    """
    Smoke test for build_codehierarchy_spans helpers:
      - load_metadata parses a small metadata file,
      - build_ontology assigns ids for languages, categories, repos,
      - build_span_records produces spans with expected node_path layout.
    """
    # Create a fake repos_root with two repos and one file each.
    repos_root = tmp_path / "repos"
    repo1 = repos_root / "repo1"
    repo2 = repos_root / "repo2"
    repo1.mkdir(parents=True)
    repo2.mkdir(parents=True)
    (repo1 / "file1.py").write_text("print('hello')\n", encoding="utf-8")
    (repo2 / "file2.cpp").write_text("int main() { return 0; }\n", encoding="utf-8")

    meta_path = tmp_path / "meta.csv"
    with meta_path.open("w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=["repo_name", "language", "category"])
        writer.writeheader()
        writer.writerow({"repo_name": "repo1", "language": "Python", "category": "ML"})
        writer.writerow({"repo_name": "repo2", "language": "C++", "category": "Systems"})

    metas = build_codehierarchy_spans.load_metadata(meta_path)
    assert len(metas) == 2

    lang_ids, cat_ids, repo_ids = build_codehierarchy_spans.build_ontology(metas)
    assert len(lang_ids) == 2
    assert len(cat_ids) == 2
    assert len(repo_ids) == 2

    spans = build_codehierarchy_spans.build_span_records(
        repos_root=repos_root,
        metas=metas,
        lang_ids=lang_ids,
        cat_ids=cat_ids,
        repo_ids=repo_ids,
    )
    # One span per file.
    assert len(spans) == 2
    for rec in spans:
        path = rec["node_path"]
        assert isinstance(path, list)
        # root + language + category + repo
        assert len(path) == 4
        assert path[0] == 0

def test_build_joint_code_paper_tol_helpers(tmp_path: Path):
    """
    Smoke test for build_joint_code_paper_tol core helpers:
      - _build_joint_ontology merges two small ontologies,
      - _rewrite_spans remaps node_path ids into the joint space,
      - _write_level_sizes produces a dict[level -> count] excluding root.
    """
    # Build minimal code and paper nodes.jsonl
    code_nodes_path = tmp_path / "code_nodes.jsonl"
    paper_nodes_path = tmp_path / "paper_nodes.jsonl"

    def _write_nodes(path: Path, levels: Dict[int, int]) -> None:
        with path.open("w", encoding="utf-8") as f:
            # local root
            f.write(
                json.dumps(
                    {
                        "node_id": 0,


    assert len(cat_ids) == 2
    assert len(repo_ids) == 2

    spans = build_codehierarchy_spans.build_span_records(
        repos_root=repos_root,
        metas=metas,
        lang_ids=lang_ids,
        cat_ids=cat_ids,
        repo_ids=repo_ids,
    )
    # One span per file.
    assert len(spans) == 2
    for rec in spans:
        path = rec["node_path"]
        assert isinstance(path, list)
        # root + language + category + repo
        assert len(path) == 4
        assert path[0] == 0

def test_build_joint_code_paper_tol_helpers(tmp_path: Path):
    """
    Smoke test for build_joint_code_paper_tol core helpers:
      - _build_joint_ontology merges two small ontologies,
      - _rewrite_spans remaps node_path ids into the joint space,
      - _write_level_sizes produces a dict[level -> count] excluding root.
    """
    # Build minimal code and paper nodes.jsonl
    code_nodes_path = tmp_path / "code_nodes.jsonl"
    paper_nodes_path = tmp_path / "paper_nodes.jsonl"

    def _write_nodes(path: Path, levels: Dict[int, int]) -> None:
        with path.open("w", encoding="utf-8") as f:
            # local root
            f.write(
                json.dumps(
                    {
                        "node_id": 0,
                        "level": 0,
                        "type": "root",
                        "parent_id": None,
                        "name": "Root",
                        "attributes": {},
                    }
                )
                + "\n"
            )
            nid = 1
            for lvl, count in levels.items():
                for i in range(count):
                    parent_id = 0 if lvl == 1 else nid - 1
                    f.write(
                        json.dumps(
                            {
                                "node_id": nid,
                                "level": lvl,
                                "type": f"lvl{lvl}",
                                "parent_id": parent_id,
                                "name": f"n{lvl}_{i}",
                                "attributes": {},
                            }
                        )
                        + "\n"
                    )
                    nid += 1

    _write_nodes(code_nodes_path, {1: 1, 2: 1})
    _write_nodes(paper_nodes_path, {1: 1, 2: 1})

    code_nodes = build_joint_code_paper_tol._load_nodes(code_nodes_path)
    paper_nodes = build_joint_code_paper_tol._load_nodes(paper_nodes_path)

    joint_nodes, code_map, paper_map, level_counts = build_joint_code_paper_tol._build_joint_ontology(
        code_nodes=code_nodes,
        paper_nodes=paper_nodes,
        code_domain_name="Code",
        paper_domain_name="Papers",
    )
    # Root + 2 domain nodes + remapped nodes from both ontologies.
    assert 0 in joint_nodes
    assert level_counts[1] == 2  # Code, Papers
    # There should be some nodes at deeper levels.
    assert any(lvl > 1 for lvl in level_counts.keys())
    assert code_map and paper_map

    # Create minimal spans that refer to old node ids and ensure they get remapped.
    code_spans_path = tmp_path / "code_spans.jsonl"
    paper_spans_path = tmp_path / "paper_spans.jsonl"
    with code_spans_path.open("w", encoding="utf-8") as f:
        f.write(json.dumps({"span_id": "c1", "text": "code", "node_path": [0, 1, 2]}) + "\n")
    with paper_spans_path.open("w", encoding="utf-8") as f:
        f.write(json.dumps({"span_id": "p1", "text": "paper", "node_path": [0, 1, 2]}) + "\n")

    # Use internal helpers directly.
    code_domain_id = 1  # first domain under root in joint ontology
    paper_domain_id = 2  # second domain
    remapped_code_spans = build_joint_code_paper_tol._rewrite_spans(
        code_spans_path, domain_node_id=code_domain_id, id_map=code_map
    )
    remapped_paper_spans = build_joint_code_paper_tol._rewrite_spans(
        paper_spans_path, domain_node_id=paper_domain_id, id_map=paper_map
    )

    assert remapped_code_spans[0]["node_path"][0] == 0
    assert remapped_code_spans[0]["node_path"][1] == code_domain_id
    assert remapped_paper_spans[0]["node_path"][0] == 0
    assert remapped_paper_spans[0]["node_path"][1] == paper_domain_id

    # Check level_sizes writing.
    level_sizes_path = tmp_path / "level_sizes.json"
    build_joint_code_paper_tol._write_level_sizes(level_counts, level_sizes_path)
    contents = json.loads(level_sizes_path.read_text(encoding="utf-8"))
    assert "level_sizes" in contents
    # Root (0) should not be present.
    assert 0 not in contents["level_sizes"]


    assert len(spans) == 2
    for rec in spans:
        path = rec["node_path"]
        assert isinstance(path, list)
        # root + language + category + repo
        assert len(path) == 4
        assert path[0] == 0

def test_build_joint_code_paper_tol_helpers(tmp_path: Path):
    """
    Smoke test for build_joint_code_paper_tol core helpers:
      - _build_joint_ontology merges two small ontologies,
      - _rewrite_spans remaps node_path ids into the joint space,
      - _write_level_sizes produces a dict[level -> count] excluding root.
    """
    # Build minimal code and paper nodes.jsonl
    code_nodes_path = tmp_path / "code_nodes.jsonl"
    paper_nodes_path = tmp_path / "paper_nodes.jsonl"

    def _write_nodes(path: Path, levels: Dict[int, int]) -> None:
        with path.open("w", encoding="utf-8") as f:
            # local root
            f.write(
                json.dumps(
                    {
                        "node_id": 0,
                        "level": 0,
                        "type": "root",
                        "parent_id": None,
                        "name": "Root",
                        "attributes": {},
                    }
                )
                + "\n"
            )
            nid = 1
            for lvl, count in levels.items():
                for i in range(count):
                    parent_id = 0 if lvl == 1 else nid - 1
                    f.write(
                        json.dumps(
                            {
                                "node_id": nid,
                                "level": lvl,
                                "type": f"lvl{lvl}",
                                "parent_id": parent_id,
                                "name": f"n{lvl}_{i}",
                                "attributes": {},
                            }
                        )
                        + "\n"
                    )
                    nid += 1

    _write_nodes(code_nodes_path, {1: 1, 2: 1})
    _write_nodes(paper_nodes_path, {1: 1, 2: 1})

    code_nodes = build_joint_code_paper_tol._load_nodes(code_nodes_path)
    paper_nodes = build_joint_code_paper_tol._load_nodes(paper_nodes_path)

    joint_nodes, code_map, paper_map, level_counts = build_joint_code_paper_tol._build_joint_ontology(
        code_nodes=code_nodes,
        paper_nodes=paper_nodes,
        code_domain_name="Code",
        paper_domain_name="Papers",
    )
    # Root + 2 domain nodes + remapped nodes from both ontologies.
    assert 0 in joint_nodes
    assert level_counts[1] == 2  # Code, Papers
    # There should be some nodes at deeper levels.
    assert any(lvl > 1 for lvl in level_counts.keys())
    assert code_map and paper_map


import torch
from pathlib import Path
import sys

# Ensure project root is on sys.path so `tolbert` imports resolve consistently.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from tolbert import losses
from tolbert import decoding

def test_tree_contrastive_loss_zero_when_no_shared_ancestors():
    """
    If no pairs share any non-root ancestor, the contrastive loss should be 0.
    """
    emb = torch.randn(3, 8)
    # All paths differ immediately after root.
    paths = [
        [0, 1],
        [0, 2],
        [0, 3],
    ]
    loss = losses.tree_contrastive_loss(emb, paths, temperature=0.1)
    assert torch.isclose(loss, torch.tensor(0.0), atol=1e-6)

def test_tree_contrastive_loss_positive_with_shared_depth():
    """
    When some pairs share deeper ancestors, the loss should be strictly positive.
    """
    # Make embeddings small and deterministic for stability.
    emb = torch.eye(4, dtype=torch.float32)
    # First three share the same label at depth 1; last one is in a different branch.
    paths = [
        [0, 1],
        [0, 1],
        [0, 1],
        [0, 2],
    ]
    loss = losses.tree_contrastive_loss(emb, paths, temperature=0.1)
    # Loss should be finite and non-negative.
    assert torch.isfinite(loss)
    assert loss.item() >= 0.0

def test_greedy_hierarchical_decode_respects_parent_child_mapping():
    """
    Ensure that greedy_hierarchical_decode:
      - performs unconstrained argmax at the first level,
      - then restricts the next level to children of the chosen parent.
    """
    batch_size = 1

    # Level 1: two parents, prefer index 1.
    l1_logits = torch.tensor([[0.0, 5.0]])  # (1, 2)
    # Level 2: three children; unconstrained argmax would pick index 2.
    l2_logits = torch.tensor([[10.0, 0.0, 1.0]])  # (1, 3)

    level_logits = {"1": l1_logits, "2": l2_logits}

    # Parent-to-children mapping: at level 2, parent 1 can only choose child 0.
    parent_to_children = {
        2: {
            0: [1, 2],
            1: [0],
        }
    }

    preds = decoding.greedy_hierarchical_decode(
        level_logits=level_logits,
        parent_to_children=parent_to_children,
        levels=[1, 2],
    )

    # Level 1: picks parent 1.
    assert preds[1].tolist() == [1]
    # Level 2: must choose from children of parent 1, i.e., [0]; so prediction is 0,
    # even though the global argmax of l2 logits is index 2.
    assert preds[2].tolist() == [0]


import torch
from pathlib import Path
import sys

# Ensure project root is on sys.path so `tolbert` imports resolve consistently.
PROJECT_ROOT = Path(__file__).resolve().parents[1]
if str(PROJECT_ROOT) not in sys.path:
    sys.path.insert(0, str(PROJECT_ROOT))

from tolbert import losses
from tolbert import decoding

def test_tree_contrastive_loss_zero_when_no_shared_ancestors():
    """
    If no pairs share any non-root ancestor, the contrastive loss should be 0.
    """
    emb = torch.randn(3, 8)
    # All paths differ immediately after root.
    paths = [
        [0, 1],
        [0, 2],
        [0, 3],
    ]
    loss = losses.tree_contrastive_loss(emb, paths, temperature=0.1)
    assert torch.isclose(loss, torch.tensor(0.0), atol=1e-6)

def test_tree_contrastive_loss_positive_with_shared_depth():
    """
    When some pairs share deeper ancestors, the loss should be strictly positive.
    """
    # Make embeddings small and deterministic for stability.
    emb = torch.eye(4, dtype=torch.float32)
    # First three share the same label at depth 1; last one is in a different branch.
    paths = [
        [0, 1],
        [0, 1],
        [0, 1],
        [0, 2],
    ]
    loss = losses.tree_contrastive_loss(emb, paths, temperature=0.1)
    # Loss should be finite and non-negative.
    assert torch.isfinite(loss)
    assert loss.item() >= 0.0


from tolbert import losses
from tolbert import decoding

def test_tree_contrastive_loss_zero_when_no_shared_ancestors():
    """
    If no pairs share any non-root ancestor, the contrastive loss should be 0.
    """
    emb = torch.randn(3, 8)
    # All paths differ immediately after root.
    paths = [
        [0, 1],
        [0, 2],
        [0, 3],
    ]
    loss = losses.tree_contrastive_loss(emb, paths, temperature=0.1)
    assert torch.isclose(loss, torch.tensor(0.0), atol=1e-6)

def test_tree_contrastive_loss_positive_with_shared_depth():
    """
    When some pairs share deeper ancestors, the loss should be strictly positive.
    """
    # Make embeddings small and deterministic for stability.
    emb = torch.eye(4, dtype=torch.float32)
    # First three share the same label at depth 1; last one is in a different branch.
    paths = [
        [0, 1],
        [0, 1],
        [0, 1],
        [0, 2],
    ]
    loss = losses.tree_contrastive_loss(emb, paths, temperature=0.1)
    # Loss should be finite and non-negative.
    assert torch.isfinite(loss)
    assert loss.item() >= 0.0

def test_greedy_hierarchical_decode_respects_parent_child_mapping():
    """
    Ensure that greedy_hierarchical_decode:
      - performs unconstrained argmax at the first level,
      - then restricts the next level to children of the chosen parent.
    """
    batch_size = 1

    # Level 1: two parents, prefer index 1.
    l1_logits = torch.tensor([[0.0, 5.0]])  # (1, 2)
    # Level 2: three children; unconstrained argmax would pick index 2.
    l2_logits = torch.tensor([[10.0, 0.0, 1.0]])  # (1, 3)

    level_logits = {"1": l1_logits, "2": l2_logits}

    # Parent-to-children mapping: at level 2, parent 1 can only choose child 0.
    parent_to_children = {
        2: {


def test_tree_contrastive_loss_positive_with_shared_depth():
    """
    When some pairs share deeper ancestors, the loss should be strictly positive.
    """
    # Make embeddings small and deterministic for stability.
    emb = torch.eye(4, dtype=torch.float32)
    # First three share the same label at depth 1; last one is in a different branch.
    paths = [
        [0, 1],
        [0, 1],
        [0, 1],
        [0, 2],
    ]
    loss = losses.tree_contrastive_loss(emb, paths, temperature=0.1)
    # Loss should be finite and non-negative.
    assert torch.isfinite(loss)
    assert loss.item() >= 0.0

def test_greedy_hierarchical_decode_respects_parent_child_mapping():
    """
    Ensure that greedy_hierarchical_decode:
      - performs unconstrained argmax at the first level,
      - then restricts the next level to children of the chosen parent.
    """
    batch_size = 1

    # Level 1: two parents, prefer index 1.
    l1_logits = torch.tensor([[0.0, 5.0]])  # (1, 2)
    # Level 2: three children; unconstrained argmax would pick index 2.
    l2_logits = torch.tensor([[10.0, 0.0, 1.0]])  # (1, 3)

    level_logits = {"1": l1_logits, "2": l2_logits}

    # Parent-to-children mapping: at level 2, parent 1 can only choose child 0.
    parent_to_children = {
        2: {
            0: [1, 2],
            1: [0],
        }
    }

    preds = decoding.greedy_hierarchical_decode(
        level_logits=level_logits,
        parent_to_children=parent_to_children,
        levels=[1, 2],
    )

    # Level 1: picks parent 1.
    assert preds[1].tolist() == [1]
    # Level 2: must choose from children of parent 1, i.e., [0]; so prediction is 0,
    # even though the global argmax of l2 logits is index 2.
    assert preds[2].tolist() == [0]


tolbert.config

import json
from pathlib import Path
from typing import Any, Dict

def load_tolbert_config(path: str) -> Dict[str, Any]:
    """
    Load a simple YAML or JSON config file into a Python dict.

    - If the extension is .yaml or .yml, this function requires PyYAML.
    - If the extension is .json, it uses the standard library.
    """
    cfg_path = Path(path)
    if not cfg_path.exists():
        raise FileNotFoundError(f"Config file not found: {cfg_path}")

    suffix = cfg_path.suffix.lower()
    if suffix in {".yaml", ".yml"}:
        try:
            import yaml  # type: ignore
        except ImportError as e:
            raise ImportError(
                "PyYAML is required to load YAML configs. Install with `pip install pyyaml`."
            ) from e

        with cfg_path.open("r", encoding="utf-8") as f:
            return yaml.safe_load(f)

    if suffix == ".json":
        with cfg_path.open("r", encoding="utf-8") as f:
            return json.load(f)

    raise ValueError(f"Unsupported config extension: {suffix} (expected .yaml, .yml, or .json)")


tolbert.modeling

from dataclasses import dataclass
from typing import Any, Dict, Optional, Sequence

import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import AutoModel

@dataclass
class TOLBERTConfig:
    """
    Minimal configuration for the TOLBERT encoder.

    This mirrors the sketch in `docs/api_reference.md` and is intended
    to be extended with any additional hyperparameters you need.
    """

    base_model_name: str
    level_sizes: Dict[int, int]
    proj_dim: int = 256
    # Overall weight on hierarchical classification loss
    lambda_hier: float = 1.0
    # Weight on path-consistency regularization term
    lambda_path: float = 0.0
    # Kept for completeness; used in the training script when adding contrastive loss.
    lambda_contrast: float = 0.0

class TOLBERT(nn.Module):
    """
    BERT/RoBERTa-style encoder with multi-level heads and a projection head.

    Forward returns:
      - loss: aggregated loss (MLM + hierarchical + optional path) if labels provided.
      - loss_components: dict with individual loss terms.
      - mlm_logits: token-level logits for MLM.
      - level_logits: dict[level_str] -> logits over nodes at that level.
      - proj: normalized CLS projection for contrastive / retrieval use.
    """

    def __init__(self, config: TOLBERTConfig):
        super().__init__()
        self.config = config

        # Backbone encoder
        self.encoder = AutoModel.from_pretrained(
            config.base_model_name,
            cache_dir="/data/checkpoints/",  # noqa: E501
        )
        hidden_dim = self.encoder.config.hidden_size

        # Simple MLM head (you can replace with the base model's own head)
        self.mlm_head = nn.Linear(hidden_dim, self.encoder.config.vocab_size)

        # Hierarchical heads per level
        self.level_heads = nn.ModuleDict()
        for level, size in config.level_sizes.items():
            self.level_heads[str(level)] = nn.Linear(hidden_dim, size)

        # Contrastive projection head
        self.proj = nn.Linear(hidden_dim, config.proj_dim)

        # Pre-sort levels once for stable ordering in losses
        self._sorted_levels = sorted(config.level_sizes.keys())

    @classmethod
    def from_pretrained(
        cls,
        checkpoint: str,
        config: Optional[TOLBERTConfig] = None,
    ) -> "TOLBERT":
        """
        Very minimal `from_pretrained` helper.

        - `checkpoint` is expected to be a `torch.save(model.state_dict())` file.
        - You must supply a `config` that matches the saved model.
        """
        if config is None:
            raise ValueError("TOLBERT.from_pretrained requires a TOLBERTConfig.")

        model = cls(config)
        state_dict = torch.load(checkpoint, map_location="cpu")
        model.load_state_dict(state_dict)
        return model

    def _compute_path_consistency_loss(
        self,
        level_logits: Dict[str, torch.Tensor],
        paths: Optional[Sequence[Sequence[int]] | Sequence[Sequence[Sequence[int]]]],
    ) -> Optional[torch.Tensor]:
        """
        Batch-local implementation of the path-consistency loss described
        in the paper / training docs, extended to handle DAGs / multi-parent
        ontologies.

        For each adjacent level pair (ℓ-1, ℓ), we:
          - build a parent -> {child,...} mapping from the observed `paths`
            across the batch (allowing multiple parents per child),
          - compute softmax over level-ℓ logits,
          - roll level-ℓ probabilities up to the parent space and
            encourage agreement between:
                - the distribution implied by children, and
                - the model's own distribution at level ℓ-1.

        In a pure tree, each child has exactly one parent; in a DAG, a child
        may appear under multiple parents across the provided paths. This
        implementation treats all observed parent-child relations as valid.
        """
        if paths is None:
            return None

        if len(self._sorted_levels) < 2:
            return None

        device = next(self.parameters()).device

        # Normalize paths into per-example path sets to support DAGs.
        # Each element in `path_sets` is a list of paths (lists of node ids).
        path_sets: list[list[list[int]]] = []
        for p in paths:
            if not p:
                path_sets.append([])
                continue
            first = p[0]
            if isinstance(first, int):
                path_sets.append([[int(x) for x in p]])  # type: ignore[arg-type]
            else:
                path_set: list[list[int]] = []
                for sub in p:  # type: ignore[assignment]
                    if isinstance(sub, (list, tuple)):
                        path_set.append([int(x) for x in sub])
                path_sets.append(path_set)

        batch_size = len(path_sets)

        total_kl = 0.0
        num_terms = 0

        # We assume level index ℓ in the paper corresponds to path index ℓ,
        # with ℓ = 1..L (and index 0 being the root, which we do not have a head for).
        # For model heads we use integer "levels" taken from config.level_sizes.
        for idx in range(1, len(self._sorted_levels)):
            level_prev = self._sorted_levels[idx - 1]
            level_curr = self._sorted_levels[idx]

            if str(level_prev) not in level_logits or str(level_curr) not in level_logits:
                continue

            logits_prev = level_logits[str(level_prev)]  # (B, C_prev)
            logits_curr = level_logits[str(level_curr)]  # (B, C_curr)

            probs_prev = F.softmax(logits_prev, dim=-1)  # (B, C_prev)
            probs_curr = F.softmax(logits_curr, dim=-1)  # (B, C_curr)

            # Build parent -> children map for this level-pair from batch paths.
            parent_to_children: Dict[int, set[int]] = {}
            for b in range(batch_size):
                for path in path_sets[b]:
                    if len(path) <= max(level_prev, level_curr):
                        continue
                    parent_id = path[level_prev]
                    child_id = path[level_curr]
                    if parent_id < 0 or child_id < 0:
                        continue
                    if parent_id not in parent_to_children:
                        parent_to_children[parent_id] = set()
                    parent_to_children[parent_id].add(child_id)

            if not parent_to_children:
                continue

            num_parents = logits_prev.size(-1)
            rolled = torch.zeros_like(probs_prev)
            for parent_id, children_ids in parent_to_children.items():
                if parent_id < 0 or parent_id >= num_parents:
                    continue
                children_idx = torch.tensor(
                    list(children_ids),
                    dtype=torch.long,
                    device=device,
                )
                children_idx = children_idx[
                    (children_idx >= 0) & (children_idx < probs_curr.size(-1))
                ]
                if children_idx.numel() == 0:
                    continue
                rolled[:, parent_id] = probs_curr[:, children_idx].sum(dim=-1)

            # Renormalize rolled-up distribution to avoid degenerate zeros.
            rolled_sum = rolled.sum(dim=-1, keepdim=True)
            rolled_sum = torch.clamp(rolled_sum, min=1e-8)
            rolled = rolled / rolled_sum

            # Ensure numerical stability for KL
            probs_prev_clamped = torch.clamp(probs_prev, min=1e-8)
            rolled_clamped = torch.clamp(rolled, min=1e-8)

            kl = F.kl_div(
                probs_prev_clamped.log(),
                rolled_clamped,
                reduction="batchmean",
            )
            total_kl = total_kl + kl
            num_terms += 1

        if num_terms == 0:
            return None

        return total_kl / float(num_terms)

    def forward(
        self,
        input_ids: torch.Tensor,
        attention_mask: Optional[torch.Tensor] = None,
        labels_mlm: Optional[torch.Tensor] = None,
        level_targets: Optional[Dict[int, torch.Tensor]] = None,
        paths: Optional[Sequence[Sequence[int]] | Sequence[Sequence[Sequence[int]]]] = None,
    ) -> Dict[str, Any]:
        """
        Args:
            input_ids: (batch, seq_len)
            attention_mask: (batch, seq_len)
            labels_mlm: (batch, seq_len) with -100 for non-MLM positions.
            level_targets: dict {level(int): tensor(batch,)} with class indices.
            paths: optional sequence of per-example label paths, each a
                   sequence of node ids [root, c1, c2, ..., cL]. Used for
                   path-consistency loss.
        """
        outputs = self.encoder(input_ids=input_ids, attention_mask=attention_mask)
        hidden_states = outputs.last_hidden_state  # (batch, seq, hidden)
        cls = hidden_states[:, 0, :]  # (batch, hidden)

        # MLM logits
        mlm_logits = self.mlm_head(hidden_states)  # (batch, seq, vocab)

        # Hierarchical logits
        level_logits: Dict[str, torch.Tensor] = {}
        for level, head in self.level_heads.items():
            level_logits[level] = head(cls)  # (batch, C_level)

        # Contrastive projection
        proj = F.normalize(self.proj(cls), dim=-1)

        loss: Optional[torch.Tensor] = None
        loss_dict: Dict[str, torch.Tensor] = {}

        # MLM loss
        mlm_loss = None
        if labels_mlm is not None:
            mlm_loss_fct = nn.CrossEntropyLoss(ignore_index=-100)
            mlm_loss = mlm_loss_fct(
                mlm_logits.view(-1, mlm_logits.size(-1)),
                labels_mlm.view(-1),
            )
            loss_dict["mlm"] = mlm_loss

        # Hierarchical classification loss (sum over levels, excluding ignore_index)
        hier_loss = None
        if level_targets is not None and len(level_targets) > 0:
            ce = nn.CrossEntropyLoss(ignore_index=-100)
            per_level_losses = []
            for level_int in self._sorted_levels:
                if level_int not in level_targets:
                    continue
                logits = level_logits[str(level_int)]
                targets = level_targets[level_int]
                level_loss = ce(logits, targets)
                loss_dict[f"level_{level_int}"] = level_loss
                per_level_losses.append(level_loss)

            if per_level_losses:
                hier_loss = torch.stack(per_level_losses).mean()
                loss_dict["hier"] = hier_loss

        # Path-consistency loss (uses predicted distributions only)
        path_loss = None
        if paths is not None:
            path_loss = self._compute_path_consistency_loss(level_logits, paths)
            if path_loss is not None:
                loss_dict["path"] = path_loss

        # Aggregate total loss with configuration weights.
        components = []
        if mlm_loss is not None:
            components.append(mlm_loss)
        if hier_loss is not None and self.config.lambda_hier != 0.0:
            components.append(self.config.lambda_hier * hier_loss)
        if path_loss is not None and self.config.lambda_path != 0.0:
            components.append(self.config.lambda_path * path_loss)

        if components:
            loss = sum(components)

        return {
            "loss": loss,
            "loss_components": loss_dict,
            "mlm_logits": mlm_logits,
            "level_logits": level_logits,
            "proj": proj,
        }