"""
Evaluate embeddings on retrieval-style tasks.

This script computes:
  - MRR (Mean Reciprocal Rank)
  - Precision@K

for a simple setup where:
  - The index consists of spans from one spans_file (e.g., code files).
  - The queries are spans from another spans_file (e.g., paper abstracts).
  - "Relevance" is defined by sharing one or more levels in the hierarchy.

By default we use "share at least level 2" as the relevance criterion, but
this can be adjusted via flags.

Usage (Paper2Code-style example):

  python -m scripts.eval_retrieval \\
      --config configs/codehierarchy_example.yaml \\
      --checkpoint checkpoints/tolbert_epoch5.pt \\
      --index-spans data/codehierarchy/spans_code.jsonl \\
      --query-spans data/wos/spans_papers.jsonl \\
      --relevant-min-level 2 \\
      --k 5
"""

from __future__ import annotations

import argparse
from pathlib import Path
from typing import Any, Dict, List, Sequence, Tuple

import torch
from torch.nn import functional as F
from transformers import AutoModel, AutoTokenizer
import os

from tolbert.config import load_tolbert_config
from tolbert.data import TreeOfLifeDataset
from tolbert.modeling import TOLBERT, TOLBERTConfig

def build_tolbert_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def encode_spans_tolbert(
    model: TOLBERT,
    tokenizer,
    spans_file: str,
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Encode all spans in `spans_file` into TOLBERT embeddings (proj head) and
    return (emb_mat, raw_records).

    This mirrors the logic in scripts/retrieval_sandbox.py but returns the full
    embedding matrix for evaluation purposes.
    """
    dataset = TreeOfLifeDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=max_length,
    )

    embs: List[torch.Tensor] = []
    metas: List[Dict[str, Any]] = []

    for rec in dataset._records:  # type: ignore[attr-defined]
        tokens = tokenizer(
            rec.text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        input_ids = tokens["input_ids"].to(device)
        attention_mask = tokens["attention_mask"].to(device)
        with torch.no_grad():
            out = model(input_ids=input_ids, attention_mask=attention_mask)
        embs.append(out["proj"].squeeze(0).cpu())
        metas.append(rec.raw)

    if not embs:
        return torch.empty(0), metas
    return torch.stack(embs, dim=0), metas

def encode_spans_hf_encoder(
    model: AutoModel,
    tokenizer,
    spans_file: str,
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Encode all spans using a vanilla HF encoder (e.g., BERT / CodeBERT /
    ModernBERT) and return CLS embeddings.

    This is used for baseline retrieval comparisons where we do not have
    TOLBERT's projection head and simply use the encoder's [CLS] representation.
    """
    dataset = TreeOfLifeDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=max_length,
    )

    embs: List[torch.Tensor] = []
    metas: List[Dict[str, Any]] = []

    for rec in dataset._records:  # type: ignore[attr-defined]
        tokens = tokenizer(
            rec.text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        input_ids = tokens["input_ids"].to(device)
        attention_mask = tokens["attention_mask"].to(device)
        with torch.no_grad():
            out = model(input_ids=input_ids, attention_mask=attention_mask)
        # Standard HF encoder output: last_hidden_state[:, 0, :] is [CLS]
        cls = out.last_hidden_state[:, 0, :]
        embs.append(cls.squeeze(0).cpu())
        metas.append(rec.raw)

    if not embs:
        return torch.empty(0), metas
    return torch.stack(embs, dim=0), metas

def compute_relevance_mask(
    query_paths: Sequence[Sequence[int]],
    index_paths: Sequence[Sequence[int]],
    min_level: int,
) -> List[List[bool]]:
    """
    Build a boolean matrix R where R[i][j] is True if query i and index j
    share at least one node at depth >= min_level in their node_path.

    Paths are lists of node ids [root, c1, c2, ...].
    """
    rel: List[List[bool]] = []
    for q_path in query_paths:
        row: List[bool] = []
        for idx_path in index_paths:
            # shared depth = deepest k where q_path[k] == idx_path[k]
            shared = False
            max_shared_level = min(len(q_path), len(idx_path)) - 1
            for lvl in range(min_level, max_shared_level + 1):
                if q_path[lvl] == idx_path[lvl]:
                    shared = True
                    break
            row.append(shared)
        rel.append(row)
    return rel

def eval_retrieval(
    query_embs: torch.Tensor,
    index_embs: torch.Tensor,
    relevant: List[List[bool]],
    k: int,
) -> Tuple[float, float]:
    """
    Compute MRR and Precision@k given:
      - query_embs: (Q, D)
      - index_embs: (N, D)
      - relevant:   QxN boolean matrix (relevance marks)
    """
    if query_embs.numel() == 0 or index_embs.numel() == 0:
        return 0.0, 0.0

    # Normalize embeddings for cosine similarity
    query_embs = F.normalize(query_embs, dim=-1)
    index_embs = F.normalize(index_embs, dim=-1)

    # Similarity matrix (Q, N)
    sims = torch.matmul(query_embs, index_embs.T)

    Q, N = sims.shape
    k = min(k, N)

    mrr = 0.0
    prec_at_k = 0.0

    for i in range(Q):
        rel_row = relevant[i]
        if not any(rel_row):
            continue  # skip queries with no relevant items

        scores = sims[i]
        topk_vals, topk_idx = torch.topk(scores, k=k)
        topk_idx_list = topk_idx.tolist()

        # MRR: find rank of first relevant
        rr = 0.0
        for rank, j in enumerate(topk_idx_list, start=1):
            if rel_row[j]:
                rr = 1.0 / rank
                break
        mrr += rr

        # Precision@k: fraction of top-k that are relevant
        num_rel_in_topk = sum(1 for j in topk_idx_list if rel_row[j])
        prec_at_k += num_rel_in_topk / float(k)

    # Normalize by number of queries that had at least one relevant item.
    num_queries_with_rel = sum(1 for row in relevant if any(row))
    if num_queries_with_rel == 0:
        return 0.0, 0.0

    mrr /= num_queries_with_rel
    prec_at_k /= num_queries_with_rel
    return mrr, prec_at_k

def compute_branch_consistency_at_k(
    query_embs: torch.Tensor,
    index_embs: torch.Tensor,
    query_paths: Sequence[Sequence[int]],
    index_paths: Sequence[Sequence[int]],
    k: int,
) -> Dict[int, float]:
    """
    Compute branch-consistency@k per hierarchy depth.

    For each query and each depth level ℓ (index in node_path, 0=root),
    we measure the fraction of the top-k retrieved items whose node_path
    matches the query's node at depth ℓ. Results are averaged over all
    queries that have a node defined at that depth.
    """
    if query_embs.numel() == 0 or index_embs.numel() == 0:
        return {}

    # Normalize embeddings for cosine similarity
    query_embs = F.normalize(query_embs, dim=-1)
    index_embs = F.normalize(index_embs, dim=-1)

    sims = torch.matmul(query_embs, index_embs.T)  # (Q, N)
    Q, N = sims.shape
    k = min(k, N)

    # Determine maximum depth observed across all paths (0-based index).
    max_depth = 0
    for p in list(query_paths) + list(index_paths):
        if p:
            max_depth = max(max_depth, len(p) - 1)

    # Accumulators: depth -> (sum_fraction, num_queries_with_node)
    num_levels = max_depth + 1
    sum_frac: List[float] = [0.0 for _ in range(num_levels)]
    count_q: List[int] = [0 for _ in range(num_levels)]

    for qi in range(Q):
        q_path = list(query_paths[qi]) if qi < len(query_paths) else []
        if not q_path:
            continue

        scores = sims[qi]
        topk_vals, topk_idx = torch.topk(scores, k=k)
        topk_indices = topk_idx.tolist()

        for depth in range(1, num_levels):  # skip depth 0 (root) by default
            if len(q_path) <= depth:
                continue
            q_node = q_path[depth]
            # Count how many of the top-k share the same node at this depth.
            matches = 0
            valid_cands = 0
            for j in topk_indices:
                if j >= len(index_paths):
                    continue
                idx_path = list(index_paths[j])
                if len(idx_path) <= depth:
                    continue
                valid_cands += 1
                if idx_path[depth] == q_node:
                    matches += 1
            if valid_cands == 0:
                continue
            frac = matches / float(valid_cands)
            sum_frac[depth] += frac
            count_q[depth] += 1

    consistency: Dict[int, float] = {}
    for depth in range(1, num_levels):
        if count_q[depth] == 0:
            continue
        consistency[depth] = sum_frac[depth] / float(count_q[depth])
    return consistency

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="Evaluate retrieval (MRR, Precision@K, branch consistency).")
    ap.add_argument("--config", type=str, required=True, help="Training config used for the model.")
    ap.add_argument(
        "--checkpoint",
        type=str,
        required=True,
        help=(
            "For mode=tolbert: path to .pt checkpoint (state_dict). "
            "For mode=hf_encoder: HF model name or directory to load via AutoModel."
        ),
    )
    ap.add_argument(
        "--index-spans",
        type=str,
        required=True,
        help="Spans JSONL file used as the retrieval index (e.g., code).",
    )
    ap.add_argument(
        "--query-spans",
        type=str,
        required=True,
        help="Spans JSONL file used as queries (e.g., papers).",
    )
    ap.add_argument(
        "--relevant-min-level",
        type=int,
        default=2,
        help="Minimum level depth (in node_path) that must match to count as relevant.",
    )
    ap.add_argument(
        "--k",
        type=int,
        default=5,
        help="Top-k for Precision@K and MRR computation (default: 5).",
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    ap.add_argument(
        "--mode",
        type=str,
        default="tolbert",
        choices=["tolbert", "hf_encoder"],
        help=(
            "Embedding backend: 'tolbert' (default, uses TOLBERT proj head) "
            "or 'hf_encoder' (vanilla HF encoder CLS for baselines like BERT, "
            "SciBERT, CodeBERT, ModernBERT)."
        ),
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)
    device = torch.device(args.device)

    index_spans_path = Path(args.index_spans)
    query_spans_path = Path(args.query_spans)
    if not index_spans_path.exists():
        raise FileNotFoundError(f"index_spans not found: {index_spans_path}")
    if not query_spans_path.exists():
        raise FileNotFoundError(f"query_spans not found: {query_spans_path}")

    tokenizer = AutoTokenizer.from_pretrained(
        cfg["base_model_name"] if args.mode == "tolbert" else args.checkpoint,
        cache_dir="/data/checkpoints/",  # noqa: E501
    )

    if args.mode == "tolbert":
        model = build_tolbert_model(cfg, checkpoint=args.checkpoint, device=device)
        encode_fn = encode_spans_tolbert
    else:
        # Vanilla HF encoder baseline (BERT, SciBERT, CodeBERT, ModernBERT, etc.)
        model = AutoModel.from_pretrained(
            args.checkpoint,
            cache_dir="/data/checkpoints/",  # noqa: E501
        )
        model.to(device)
        model.eval()
        encode_fn = encode_spans_hf_encoder

    print(f"Encoding index spans from {index_spans_path} ...")
    index_embs, index_metas = encode_fn(
        model=model,
        tokenizer=tokenizer,
        spans_file=str(index_spans_path),
        max_length=cfg.get("max_length", 256),
        device=device,
    )

    print(f"Encoding query spans from {query_spans_path} ...")
    query_embs, query_metas = encode_fn(
        model=model,
        tokenizer=tokenizer,
        spans_file=str(query_spans_path),
        max_length=cfg.get("max_length", 256),
        device=device,
    )

    # Extract node_path lists from metas.
    index_paths: List[Sequence[int]] = [
        rec.get("node_path", []) for rec in index_metas  # type: ignore[assignment]
    ]
    query_paths: List[Sequence[int]] = [
        rec.get("node_path", []) for rec in query_metas  # type: ignore[assignment]
    ]

    relevant = compute_relevance_mask(
        query_paths=query_paths,
        index_paths=index_paths,
        min_level=args.relevant_min_level,
    )

    mrr, p_at_k = eval_retrieval(
        query_embs=query_embs,
        index_embs=index_embs,
        relevant=relevant,
        k=args.k,
    )

    print("=== Retrieval Evaluation ===")
    print(f"MRR:        {mrr:.4f}")
    print(f"Precision@{args.k}: {p_at_k:.4f}")

    # Ontology-aware metric: branch-consistency@k per depth level.
    branch_consistency = compute_branch_consistency_at_k(
        query_embs=query_embs,
        index_embs=index_embs,
        query_paths=query_paths,
        index_paths=index_paths,
        k=args.k,
    )
    if branch_consistency:
        print("\nBranch-consistency@{k} by depth (node_path index):".format(k=args.k))
        for depth in sorted(branch_consistency.keys()):
            # depth 0 is the root; depths >=1 correspond to actual hierarchy levels.
            print(f"  depth {depth}: {branch_consistency[depth]:.4f}")

if __name__ == "__main__":
    main()


      --relevant-min-level 2 \\
      --k 5
"""

from __future__ import annotations

import argparse
from pathlib import Path
from typing import Any, Dict, List, Sequence, Tuple

import torch
from torch.nn import functional as F
from transformers import AutoModel, AutoTokenizer
import os

from tolbert.config import load_tolbert_config
from tolbert.data import TreeOfLifeDataset
from tolbert.modeling import TOLBERT, TOLBERTConfig

def build_tolbert_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def encode_spans_tolbert(
    model: TOLBERT,
    tokenizer,
    spans_file: str,
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Encode all spans in `spans_file` into TOLBERT embeddings (proj head) and
    return (emb_mat, raw_records).

    This mirrors the logic in scripts/retrieval_sandbox.py but returns the full
    embedding matrix for evaluation purposes.
    """
    dataset = TreeOfLifeDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=max_length,


from tolbert.config import load_tolbert_config
from tolbert.data import TreeOfLifeDataset
from tolbert.modeling import TOLBERT, TOLBERTConfig

def build_tolbert_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def encode_spans_tolbert(
    model: TOLBERT,
    tokenizer,
    spans_file: str,
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Encode all spans in `spans_file` into TOLBERT embeddings (proj head) and
    return (emb_mat, raw_records).

    This mirrors the logic in scripts/retrieval_sandbox.py but returns the full
    embedding matrix for evaluation purposes.
    """
    dataset = TreeOfLifeDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=max_length,
    )

    embs: List[torch.Tensor] = []
    metas: List[Dict[str, Any]] = []

    for rec in dataset._records:  # type: ignore[attr-defined]
        tokens = tokenizer(
            rec.text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        input_ids = tokens["input_ids"].to(device)
        attention_mask = tokens["attention_mask"].to(device)
        with torch.no_grad():
            out = model(input_ids=input_ids, attention_mask=attention_mask)
        embs.append(out["proj"].squeeze(0).cpu())
        metas.append(rec.raw)

    if not embs:
        return torch.empty(0), metas
    return torch.stack(embs, dim=0), metas

def encode_spans_hf_encoder(
    model: AutoModel,
    tokenizer,
    spans_file: str,
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Encode all spans using a vanilla HF encoder (e.g., BERT / CodeBERT /
    ModernBERT) and return CLS embeddings.

    This is used for baseline retrieval comparisons where we do not have
    TOLBERT's projection head and simply use the encoder's [CLS] representation.
    """
    dataset = TreeOfLifeDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=max_length,


    for rec in dataset._records:  # type: ignore[attr-defined]
        tokens = tokenizer(
            rec.text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        input_ids = tokens["input_ids"].to(device)
        attention_mask = tokens["attention_mask"].to(device)
        with torch.no_grad():
            out = model(input_ids=input_ids, attention_mask=attention_mask)
        embs.append(out["proj"].squeeze(0).cpu())
        metas.append(rec.raw)

    if not embs:
        return torch.empty(0), metas
    return torch.stack(embs, dim=0), metas

def encode_spans_hf_encoder(
    model: AutoModel,
    tokenizer,
    spans_file: str,
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Encode all spans using a vanilla HF encoder (e.g., BERT / CodeBERT /
    ModernBERT) and return CLS embeddings.

    This is used for baseline retrieval comparisons where we do not have
    TOLBERT's projection head and simply use the encoder's [CLS] representation.
    """
    dataset = TreeOfLifeDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=max_length,
    )

    embs: List[torch.Tensor] = []
    metas: List[Dict[str, Any]] = []

    for rec in dataset._records:  # type: ignore[attr-defined]
        tokens = tokenizer(
            rec.text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        input_ids = tokens["input_ids"].to(device)
        attention_mask = tokens["attention_mask"].to(device)
        with torch.no_grad():
            out = model(input_ids=input_ids, attention_mask=attention_mask)
        # Standard HF encoder output: last_hidden_state[:, 0, :] is [CLS]
        cls = out.last_hidden_state[:, 0, :]
        embs.append(cls.squeeze(0).cpu())
        metas.append(rec.raw)

    if not embs:
        return torch.empty(0), metas
    return torch.stack(embs, dim=0), metas

def compute_relevance_mask(
    query_paths: Sequence[Sequence[int]],
    index_paths: Sequence[Sequence[int]],
    min_level: int,
) -> List[List[bool]]:
    """
    Build a boolean matrix R where R[i][j] is True if query i and index j
    share at least one node at depth >= min_level in their node_path.

    Paths are lists of node ids [root, c1, c2, ...].
    """
    rel: List[List[bool]] = []
    for q_path in query_paths:
        row: List[bool] = []
        for idx_path in index_paths:
            # shared depth = deepest k where q_path[k] == idx_path[k]
            shared = False
            max_shared_level = min(len(q_path), len(idx_path)) - 1


            rec.text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        input_ids = tokens["input_ids"].to(device)
        attention_mask = tokens["attention_mask"].to(device)
        with torch.no_grad():
            out = model(input_ids=input_ids, attention_mask=attention_mask)
        # Standard HF encoder output: last_hidden_state[:, 0, :] is [CLS]
        cls = out.last_hidden_state[:, 0, :]
        embs.append(cls.squeeze(0).cpu())
        metas.append(rec.raw)

    if not embs:
        return torch.empty(0), metas
    return torch.stack(embs, dim=0), metas

def compute_relevance_mask(
    query_paths: Sequence[Sequence[int]],
    index_paths: Sequence[Sequence[int]],
    min_level: int,
) -> List[List[bool]]:
    """
    Build a boolean matrix R where R[i][j] is True if query i and index j
    share at least one node at depth >= min_level in their node_path.

    Paths are lists of node ids [root, c1, c2, ...].
    """
    rel: List[List[bool]] = []
    for q_path in query_paths:
        row: List[bool] = []
        for idx_path in index_paths:
            # shared depth = deepest k where q_path[k] == idx_path[k]
            shared = False
            max_shared_level = min(len(q_path), len(idx_path)) - 1
            for lvl in range(min_level, max_shared_level + 1):
                if q_path[lvl] == idx_path[lvl]:
                    shared = True
                    break
            row.append(shared)
        rel.append(row)
    return rel

def eval_retrieval(
    query_embs: torch.Tensor,
    index_embs: torch.Tensor,
    relevant: List[List[bool]],
    k: int,
) -> Tuple[float, float]:
    """
    Compute MRR and Precision@k given:
      - query_embs: (Q, D)
      - index_embs: (N, D)
      - relevant:   QxN boolean matrix (relevance marks)
    """
    if query_embs.numel() == 0 or index_embs.numel() == 0:
        return 0.0, 0.0

    # Normalize embeddings for cosine similarity
    query_embs = F.normalize(query_embs, dim=-1)
    index_embs = F.normalize(index_embs, dim=-1)


    share at least one node at depth >= min_level in their node_path.

    Paths are lists of node ids [root, c1, c2, ...].
    """
    rel: List[List[bool]] = []
    for q_path in query_paths:
        row: List[bool] = []
        for idx_path in index_paths:
            # shared depth = deepest k where q_path[k] == idx_path[k]
            shared = False
            max_shared_level = min(len(q_path), len(idx_path)) - 1
            for lvl in range(min_level, max_shared_level + 1):
                if q_path[lvl] == idx_path[lvl]:
                    shared = True
                    break
            row.append(shared)
        rel.append(row)
    return rel

def eval_retrieval(
    query_embs: torch.Tensor,
    index_embs: torch.Tensor,
    relevant: List[List[bool]],
    k: int,
) -> Tuple[float, float]:
    """
    Compute MRR and Precision@k given:
      - query_embs: (Q, D)
      - index_embs: (N, D)
      - relevant:   QxN boolean matrix (relevance marks)
    """
    if query_embs.numel() == 0 or index_embs.numel() == 0:
        return 0.0, 0.0

    # Normalize embeddings for cosine similarity
    query_embs = F.normalize(query_embs, dim=-1)
    index_embs = F.normalize(index_embs, dim=-1)

    # Similarity matrix (Q, N)
    sims = torch.matmul(query_embs, index_embs.T)

    Q, N = sims.shape
    k = min(k, N)

    mrr = 0.0
    prec_at_k = 0.0

    for i in range(Q):
        rel_row = relevant[i]
        if not any(rel_row):
            continue  # skip queries with no relevant items

        scores = sims[i]
        topk_vals, topk_idx = torch.topk(scores, k=k)
        topk_idx_list = topk_idx.tolist()

        # MRR: find rank of first relevant
        rr = 0.0
        for rank, j in enumerate(topk_idx_list, start=1):
            if rel_row[j]:
                rr = 1.0 / rank
                break
        mrr += rr

        # Precision@k: fraction of top-k that are relevant
        num_rel_in_topk = sum(1 for j in topk_idx_list if rel_row[j])
        prec_at_k += num_rel_in_topk / float(k)

    # Normalize by number of queries that had at least one relevant item.
    num_queries_with_rel = sum(1 for row in relevant if any(row))
    if num_queries_with_rel == 0:
        return 0.0, 0.0

    mrr /= num_queries_with_rel
    prec_at_k /= num_queries_with_rel
    return mrr, prec_at_k

def compute_branch_consistency_at_k(
    query_embs: torch.Tensor,
    index_embs: torch.Tensor,
    query_paths: Sequence[Sequence[int]],
    index_paths: Sequence[Sequence[int]],
    k: int,
) -> Dict[int, float]:
    """
    Compute branch-consistency@k per hierarchy depth.

    For each query and each depth level ℓ (index in node_path, 0=root),
    we measure the fraction of the top-k retrieved items whose node_path
    matches the query's node at depth ℓ. Results are averaged over all
    queries that have a node defined at that depth.
    """
    if query_embs.numel() == 0 or index_embs.numel() == 0:
        return {}


        for rank, j in enumerate(topk_idx_list, start=1):
            if rel_row[j]:
                rr = 1.0 / rank
                break
        mrr += rr

        # Precision@k: fraction of top-k that are relevant
        num_rel_in_topk = sum(1 for j in topk_idx_list if rel_row[j])
        prec_at_k += num_rel_in_topk / float(k)

    # Normalize by number of queries that had at least one relevant item.
    num_queries_with_rel = sum(1 for row in relevant if any(row))
    if num_queries_with_rel == 0:
        return 0.0, 0.0

    mrr /= num_queries_with_rel
    prec_at_k /= num_queries_with_rel
    return mrr, prec_at_k

def compute_branch_consistency_at_k(
    query_embs: torch.Tensor,
    index_embs: torch.Tensor,
    query_paths: Sequence[Sequence[int]],
    index_paths: Sequence[Sequence[int]],
    k: int,
) -> Dict[int, float]:
    """
    Compute branch-consistency@k per hierarchy depth.

    For each query and each depth level ℓ (index in node_path, 0=root),
    we measure the fraction of the top-k retrieved items whose node_path
    matches the query's node at depth ℓ. Results are averaged over all
    queries that have a node defined at that depth.
    """
    if query_embs.numel() == 0 or index_embs.numel() == 0:
        return {}

    # Normalize embeddings for cosine similarity
    query_embs = F.normalize(query_embs, dim=-1)
    index_embs = F.normalize(index_embs, dim=-1)

    sims = torch.matmul(query_embs, index_embs.T)  # (Q, N)
    Q, N = sims.shape
    k = min(k, N)

    # Determine maximum depth observed across all paths (0-based index).
    max_depth = 0
    for p in list(query_paths) + list(index_paths):
        if p:
            max_depth = max(max_depth, len(p) - 1)

    # Accumulators: depth -> (sum_fraction, num_queries_with_node)
    num_levels = max_depth + 1
    sum_frac: List[float] = [0.0 for _ in range(num_levels)]
    count_q: List[int] = [0 for _ in range(num_levels)]

    for qi in range(Q):
        q_path = list(query_paths[qi]) if qi < len(query_paths) else []
        if not q_path:
            continue

        scores = sims[qi]
        topk_vals, topk_idx = torch.topk(scores, k=k)
        topk_indices = topk_idx.tolist()

        for depth in range(1, num_levels):  # skip depth 0 (root) by default
            if len(q_path) <= depth:
                continue
            q_node = q_path[depth]
            # Count how many of the top-k share the same node at this depth.
            matches = 0
            valid_cands = 0
            for j in topk_indices:
                if j >= len(index_paths):
                    continue
                idx_path = list(index_paths[j])
                if len(idx_path) <= depth:
                    continue
                valid_cands += 1
                if idx_path[depth] == q_node:
                    matches += 1
            if valid_cands == 0:
                continue
            frac = matches / float(valid_cands)
            sum_frac[depth] += frac
            count_q[depth] += 1

    consistency: Dict[int, float] = {}
    for depth in range(1, num_levels):
        if count_q[depth] == 0:
            continue
        consistency[depth] = sum_frac[depth] / float(count_q[depth])
    return consistency

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="Evaluate retrieval (MRR, Precision@K, branch consistency).")
    ap.add_argument("--config", type=str, required=True, help="Training config used for the model.")
    ap.add_argument(
        "--checkpoint",
        type=str,
        required=True,
        help=(
            "For mode=tolbert: path to .pt checkpoint (state_dict). "
            "For mode=hf_encoder: HF model name or directory to load via AutoModel."
        ),
    )
    ap.add_argument(
        "--index-spans",
        type=str,
        required=True,
        help="Spans JSONL file used as the retrieval index (e.g., code).",
    )


                idx_path = list(index_paths[j])
                if len(idx_path) <= depth:
                    continue
                valid_cands += 1
                if idx_path[depth] == q_node:
                    matches += 1
            if valid_cands == 0:
                continue
            frac = matches / float(valid_cands)
            sum_frac[depth] += frac
            count_q[depth] += 1

    consistency: Dict[int, float] = {}
    for depth in range(1, num_levels):
        if count_q[depth] == 0:
            continue
        consistency[depth] = sum_frac[depth] / float(count_q[depth])
    return consistency

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="Evaluate retrieval (MRR, Precision@K, branch consistency).")
    ap.add_argument("--config", type=str, required=True, help="Training config used for the model.")
    ap.add_argument(
        "--checkpoint",
        type=str,
        required=True,
        help=(
            "For mode=tolbert: path to .pt checkpoint (state_dict). "
            "For mode=hf_encoder: HF model name or directory to load via AutoModel."
        ),
    )
    ap.add_argument(
        "--index-spans",
        type=str,
        required=True,
        help="Spans JSONL file used as the retrieval index (e.g., code).",
    )
    ap.add_argument(
        "--query-spans",
        type=str,
        required=True,
        help="Spans JSONL file used as queries (e.g., papers).",
    )
    ap.add_argument(
        "--relevant-min-level",
        type=int,
        default=2,
        help="Minimum level depth (in node_path) that must match to count as relevant.",
    )
    ap.add_argument(
        "--k",
        type=int,
        default=5,
        help="Top-k for Precision@K and MRR computation (default: 5).",
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    ap.add_argument(
        "--mode",
        type=str,
        default="tolbert",
        choices=["tolbert", "hf_encoder"],
        help=(
            "Embedding backend: 'tolbert' (default, uses TOLBERT proj head) "
            "or 'hf_encoder' (vanilla HF encoder CLS for baselines like BERT, "
            "SciBERT, CodeBERT, ModernBERT)."
        ),
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)
    device = torch.device(args.device)

    index_spans_path = Path(args.index_spans)
    query_spans_path = Path(args.query_spans)
    if not index_spans_path.exists():
        raise FileNotFoundError(f"index_spans not found: {index_spans_path}")
    if not query_spans_path.exists():
        raise FileNotFoundError(f"query_spans not found: {query_spans_path}")

    tokenizer = AutoTokenizer.from_pretrained(
        cfg["base_model_name"] if args.mode == "tolbert" else args.checkpoint,
        cache_dir="/data/checkpoints/",  # noqa: E501
    )

    if args.mode == "tolbert":


main

    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    ap.add_argument(
        "--mode",
        type=str,
        default="tolbert",
        choices=["tolbert", "hf_encoder"],
        help=(
            "Embedding backend: 'tolbert' (default, uses TOLBERT proj head) "
            "or 'hf_encoder' (vanilla HF encoder CLS for baselines like BERT, "
            "SciBERT, CodeBERT, ModernBERT)."
        ),
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)
    device = torch.device(args.device)

    index_spans_path = Path(args.index_spans)
    query_spans_path = Path(args.query_spans)
    if not index_spans_path.exists():
        raise FileNotFoundError(f"index_spans not found: {index_spans_path}")
    if not query_spans_path.exists():
        raise FileNotFoundError(f"query_spans not found: {query_spans_path}")

    tokenizer = AutoTokenizer.from_pretrained(
        cfg["base_model_name"] if args.mode == "tolbert" else args.checkpoint,
        cache_dir="/data/checkpoints/",  # noqa: E501
    )

    if args.mode == "tolbert":
        model = build_tolbert_model(cfg, checkpoint=args.checkpoint, device=device)
        encode_fn = encode_spans_tolbert
    else:
        # Vanilla HF encoder baseline (BERT, SciBERT, CodeBERT, ModernBERT, etc.)
        model = AutoModel.from_pretrained(
            args.checkpoint,
            cache_dir="/data/checkpoints/",  # noqa: E501
        )
        model.to(device)
        model.eval()
        encode_fn = encode_spans_hf_encoder

    print(f"Encoding index spans from {index_spans_path} ...")
    index_embs, index_metas = encode_fn(
        model=model,
        tokenizer=tokenizer,
        spans_file=str(index_spans_path),
        max_length=cfg.get("max_length", 256),
        device=device,
    )

    print(f"Encoding query spans from {query_spans_path} ...")
    query_embs, query_metas = encode_fn(
        model=model,
        tokenizer=tokenizer,
        spans_file=str(query_spans_path),
        max_length=cfg.get("max_length", 256),
        device=device,
    )

    # Extract node_path lists from metas.
    index_paths: List[Sequence[int]] = [
        rec.get("node_path", []) for rec in index_metas  # type: ignore[assignment]
    ]
    query_paths: List[Sequence[int]] = [
        rec.get("node_path", []) for rec in query_metas  # type: ignore[assignment]
    ]

    relevant = compute_relevance_mask(
        query_paths=query_paths,
        index_paths=index_paths,
        min_level=args.relevant_min_level,
    )

    mrr, p_at_k = eval_retrieval(
        query_embs=query_embs,
        index_embs=index_embs,
        relevant=relevant,
        k=args.k,
    )

    print("=== Retrieval Evaluation ===")
    print(f"MRR:        {mrr:.4f}")
    print(f"Precision@{args.k}: {p_at_k:.4f}")

    # Ontology-aware metric: branch-consistency@k per depth level.
    branch_consistency = compute_branch_consistency_at_k(
        query_embs=query_embs,
        index_embs=index_embs,
        query_paths=query_paths,
        index_paths=index_paths,
        k=args.k,
    )
    if branch_consistency:
        print("\nBranch-consistency@{k} by depth (node_path index):".format(k=args.k))
        for depth in sorted(branch_consistency.keys()):
            # depth 0 is the root; depths >=1 correspond to actual hierarchy levels.
            print(f"  depth {depth}: {branch_consistency[depth]:.4f}")

if __name__ == "__main__":
    main()


"""
Train a flat (leaf-level) baseline classifier on spans_file.

This script is intended to reproduce the "BERT-flat" style baselines from
Section 4 using standard HuggingFace models (e.g. BERT, SciBERT, CodeBERT,
ModernBERT) on the same JSONL spans files used for TOLBERT.

Key characteristics:
  - Single-label, flat classification on the *leaf* node of each path
    (last element of `node_path`).
  - Uses AutoModelForSequenceClassification with a single softmax head.
  - Ignores intermediate hierarchy levels entirely.

Example (CodeHierarchy, BERT-base flat):

  python -m scripts.train_flat_baseline \\
      --config configs/codehierarchy_example.yaml \\
      --output-dir checkpoints/codehierarchy_bert_flat \\
      --base-model-name bert-base-uncased

Example (WOS, SciBERT flat):

  python -m scripts.train_flat_baseline \\
      --config configs/wos_example.yaml \\
      --output-dir checkpoints/wos_scibert_flat \\
      --base-model-name allenai/scibert_scivocab_uncased

Example (CodeHierarchy, CodeBERT flat):

  python -m scripts.train_flat_baseline \\
      --config configs/codehierarchy_example.yaml \\
      --output-dir checkpoints/codehierarchy_codebert_flat \\
      --base-model-name microsoft/codebert-base

Example (mixed-domain, ModernBERT flat):

  python -m scripts.train_flat_baseline \\
      --config configs/codehierarchy_example.yaml \\
      --output-dir checkpoints/codehierarchy_modernbert_flat \\
      --base-model-name answerdotai/ModernBERT-base
"""

from __future__ import annotations

import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

import torch
from torch.utils.data import DataLoader, Dataset
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    PreTrainedTokenizerBase,
)
import os

from tolbert.config import load_tolbert_config

class FlatLeafDataset(Dataset):
    """
    Minimal dataset for flat (leaf-level) classification.

    Expects the same JSONL format as TreeOfLifeDataset, but uses only:
      - "text": span text
      - "node_path": [root_id, ..., leaf_id]

    The label is taken to be the *last* element of node_path (leaf).
    """

    def __init__(
        self,
        spans_file: str,
        tokenizer: PreTrainedTokenizerBase,
        max_length: int = 256,
    ) -> None:
        self.spans_file = spans_file
        self.tokenizer = tokenizer
        self.max_length = max_length

        self._records: List[Dict[str, Any]] = []
        with open(spans_file, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                obj = json.loads(line)
                if "text" not in obj or "node_path" not in obj:
                    # Skip spans that do not have both text and a path.
                    continue
                if not obj["node_path"]:
                    continue
                self._records.append(obj)

    def __len__(self) -> int:
        return len(self._records)

    def _tokenize(self, text: str) -> Dict[str, torch.Tensor]:
        enc = self.tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=self.max_length,
        )
        return {k: v.squeeze(0) for k, v in enc.items()}

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        rec = self._records[idx]
        tokens = self._tokenize(rec["text"])
        input_ids = tokens["input_ids"]
        attention_mask = tokens.get("attention_mask", torch.ones_like(input_ids))
        # Leaf = last element of node_path
        node_path: List[int] = rec["node_path"]
        label = int(node_path[-1])
        return {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "labels": torch.tensor(label, dtype=torch.long),
        }

def collate_flat_batch(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
    input_ids = torch.stack([b["input_ids"] for b in batch], dim=0)
    attention_mask = torch.stack([b["attention_mask"] for b in batch], dim=0)
    labels = torch.stack([b["labels"] for b in batch], dim=0)
    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "labels": labels,
    }

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="Train a flat (leaf-level) baseline classifier.")
    ap.add_argument(
        "--config",
        type=str,
        required=True,
        help="YAML/JSON config with at least spans_file, level_sizes, batch_size, num_epochs, lr.",
    )
    ap.add_argument(
        "--output-dir",
        type=str,
        required=True,
        help="Directory to save the fine-tuned baseline model (HF save_pretrained format).",
    )
    ap.add_argument(
        "--base-model-name",
        type=str,
        required=True,
        help=(
            "HF model name or path for the baseline backbone "
            "(e.g., bert-base-uncased, allenai/scibert_scivocab_uncased, "
            "microsoft/codebert-base, answerdotai/ModernBERT-base)."
        ),
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use (default: cuda if available, else cpu).",
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)

    device = torch.device(args.device)

    spans_file = cfg["spans_file"]
    if not Path(spans_file).exists():
        raise FileNotFoundError(f"spans_file not found: {spans_file}")

    # Determine number of leaf classes from level_sizes: leaf = max level index.
    level_sizes: Dict[int, int] = cfg["level_sizes"]
    leaf_level = max(level_sizes.keys())
    num_labels = level_sizes[leaf_level]

    tokenizer = AutoTokenizer.from_pretrained(
        args.base_model_name,
        cache_dir="/data/checkpoints/",  # noqa: E501
    )
    dataset = FlatLeafDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=cfg.get("max_length", 256),
    )

    dataloader = DataLoader(
        dataset,
        batch_size=cfg.get("batch_size", 64),
        shuffle=True,
        num_workers=cfg.get("num_workers", 0),
        collate_fn=collate_flat_batch,
    )

    model = AutoModelForSequenceClassification.from_pretrained(
        args.base_model_name,
        num_labels=num_labels,
        cache_dir="/data/checkpoints/",  # noqa: E501
    )
    model.to(device)
    model.train()

    optimizer = torch.optim.AdamW(model.parameters(), lr=cfg.get("lr", 1e-4))
    num_epochs = cfg.get("num_epochs", 1)
    log_every = cfg.get("log_every", 50)

    global_step = 0
    for epoch in range(num_epochs):
        for batch in dataloader:
            global_step += 1
            optimizer.zero_grad()

            input_ids = batch["input_ids"].to(device)
            attention_mask = batch["attention_mask"].to(device)
            labels = batch["labels"].to(device)

            outputs = model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                labels=labels,
            )
            loss = outputs.loss
            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=cfg.get("grad_clip", 1.0))
            optimizer.step()

            if global_step % log_every == 0:
                print(f"[epoch {epoch+1} step {global_step}] loss={loss.item():.4f}")

    out_dir = Path(args.output_dir)
    out_dir.mkdir(parents=True, exist_ok=True)
    model.save_pretrained(out_dir)
    tokenizer.save_pretrained(out_dir)
    print(f"Saved flat baseline model to {out_dir}")

if __name__ == "__main__":
    main()


FlatLeafDataset

from __future__ import annotations

import argparse
import json
from pathlib import Path
from typing import Any, Dict, List

import torch
from torch.utils.data import DataLoader, Dataset
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    PreTrainedTokenizerBase,
)
import os

from tolbert.config import load_tolbert_config

class FlatLeafDataset(Dataset):
    """
    Minimal dataset for flat (leaf-level) classification.

    Expects the same JSONL format as TreeOfLifeDataset, but uses only:
      - "text": span text
      - "node_path": [root_id, ..., leaf_id]

    The label is taken to be the *last* element of node_path (leaf).
    """

    def __init__(
        self,
        spans_file: str,
        tokenizer: PreTrainedTokenizerBase,
        max_length: int = 256,
    ) -> None:
        self.spans_file = spans_file
        self.tokenizer = tokenizer
        self.max_length = max_length

        self._records: List[Dict[str, Any]] = []
        with open(spans_file, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                obj = json.loads(line)
                if "text" not in obj or "node_path" not in obj:
                    # Skip spans that do not have both text and a path.
                    continue
                if not obj["node_path"]:
                    continue
                self._records.append(obj)

    def __len__(self) -> int:
        return len(self._records)

    def _tokenize(self, text: str) -> Dict[str, torch.Tensor]:
        enc = self.tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=self.max_length,
        )
        return {k: v.squeeze(0) for k, v in enc.items()}

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        rec = self._records[idx]
        tokens = self._tokenize(rec["text"])
        input_ids = tokens["input_ids"]
        attention_mask = tokens.get("attention_mask", torch.ones_like(input_ids))
        # Leaf = last element of node_path
        node_path: List[int] = rec["node_path"]
        label = int(node_path[-1])
        return {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "labels": torch.tensor(label, dtype=torch.long),
        }

def collate_flat_batch(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
    input_ids = torch.stack([b["input_ids"] for b in batch], dim=0)
    attention_mask = torch.stack([b["attention_mask"] for b in batch], dim=0)
    labels = torch.stack([b["labels"] for b in batch], dim=0)
    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "labels": labels,
    }

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="Train a flat (leaf-level) baseline classifier.")
    ap.add_argument(
        "--config",
        type=str,
        required=True,
        help="YAML/JSON config with at least spans_file, level_sizes, batch_size, num_epochs, lr.",


            padding="max_length",
            max_length=self.max_length,
        )
        return {k: v.squeeze(0) for k, v in enc.items()}

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        rec = self._records[idx]
        tokens = self._tokenize(rec["text"])
        input_ids = tokens["input_ids"]
        attention_mask = tokens.get("attention_mask", torch.ones_like(input_ids))
        # Leaf = last element of node_path
        node_path: List[int] = rec["node_path"]
        label = int(node_path[-1])
        return {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "labels": torch.tensor(label, dtype=torch.long),
        }

def collate_flat_batch(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
    input_ids = torch.stack([b["input_ids"] for b in batch], dim=0)
    attention_mask = torch.stack([b["attention_mask"] for b in batch], dim=0)
    labels = torch.stack([b["labels"] for b in batch], dim=0)
    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "labels": labels,
    }

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="Train a flat (leaf-level) baseline classifier.")
    ap.add_argument(
        "--config",
        type=str,
        required=True,
        help="YAML/JSON config with at least spans_file, level_sizes, batch_size, num_epochs, lr.",
    )
    ap.add_argument(
        "--output-dir",
        type=str,
        required=True,
        help="Directory to save the fine-tuned baseline model (HF save_pretrained format).",
    )
    ap.add_argument(
        "--base-model-name",
        type=str,
        required=True,


        node_path: List[int] = rec["node_path"]
        label = int(node_path[-1])
        return {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "labels": torch.tensor(label, dtype=torch.long),
        }

def collate_flat_batch(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
    input_ids = torch.stack([b["input_ids"] for b in batch], dim=0)
    attention_mask = torch.stack([b["attention_mask"] for b in batch], dim=0)
    labels = torch.stack([b["labels"] for b in batch], dim=0)
    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "labels": labels,
    }

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="Train a flat (leaf-level) baseline classifier.")
    ap.add_argument(
        "--config",
        type=str,
        required=True,
        help="YAML/JSON config with at least spans_file, level_sizes, batch_size, num_epochs, lr.",
    )
    ap.add_argument(
        "--output-dir",
        type=str,
        required=True,
        help="Directory to save the fine-tuned baseline model (HF save_pretrained format).",
    )
    ap.add_argument(
        "--base-model-name",
        type=str,
        required=True,
        help=(
            "HF model name or path for the baseline backbone "
            "(e.g., bert-base-uncased, allenai/scibert_scivocab_uncased, "
            "microsoft/codebert-base, answerdotai/ModernBERT-base)."
        ),
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use (default: cuda if available, else cpu).",
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)

    device = torch.device(args.device)

    spans_file = cfg["spans_file"]
    if not Path(spans_file).exists():
        raise FileNotFoundError(f"spans_file not found: {spans_file}")

    # Determine number of leaf classes from level_sizes: leaf = max level index.
    level_sizes: Dict[int, int] = cfg["level_sizes"]
    leaf_level = max(level_sizes.keys())
    num_labels = level_sizes[leaf_level]

    tokenizer = AutoTokenizer.from_pretrained(
        args.base_model_name,
        cache_dir="/data/checkpoints/",  # noqa: E501


main

    )
    ap.add_argument(
        "--base-model-name",
        type=str,
        required=True,
        help=(
            "HF model name or path for the baseline backbone "
            "(e.g., bert-base-uncased, allenai/scibert_scivocab_uncased, "
            "microsoft/codebert-base, answerdotai/ModernBERT-base)."
        ),
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use (default: cuda if available, else cpu).",
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)

    device = torch.device(args.device)

    spans_file = cfg["spans_file"]
    if not Path(spans_file).exists():
        raise FileNotFoundError(f"spans_file not found: {spans_file}")

    # Determine number of leaf classes from level_sizes: leaf = max level index.
    level_sizes: Dict[int, int] = cfg["level_sizes"]
    leaf_level = max(level_sizes.keys())
    num_labels = level_sizes[leaf_level]

    tokenizer = AutoTokenizer.from_pretrained(
        args.base_model_name,
        cache_dir="/data/checkpoints/",  # noqa: E501
    )
    dataset = FlatLeafDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=cfg.get("max_length", 256),
    )

    dataloader = DataLoader(
        dataset,
        batch_size=cfg.get("batch_size", 64),
        shuffle=True,
        num_workers=cfg.get("num_workers", 0),
        collate_fn=collate_flat_batch,
    )

    model = AutoModelForSequenceClassification.from_pretrained(
        args.base_model_name,
        num_labels=num_labels,
        cache_dir="/data/checkpoints/",  # noqa: E501
    )
    model.to(device)
    model.train()

    optimizer = torch.optim.AdamW(model.parameters(), lr=cfg.get("lr", 1e-4))
    num_epochs = cfg.get("num_epochs", 1)
    log_every = cfg.get("log_every", 50)

    global_step = 0
    for epoch in range(num_epochs):
        for batch in dataloader:
            global_step += 1
            optimizer.zero_grad()

            input_ids = batch["input_ids"].to(device)
            attention_mask = batch["attention_mask"].to(device)
            labels = batch["labels"].to(device)

            outputs = model(
                input_ids=input_ids,
                attention_mask=attention_mask,
                labels=labels,
            )
            loss = outputs.loss
            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=cfg.get("grad_clip", 1.0))
            optimizer.step()

            if global_step % log_every == 0:
                print(f"[epoch {epoch+1} step {global_step}] loss={loss.item():.4f}")

    out_dir = Path(args.output_dir)
    out_dir.mkdir(parents=True, exist_ok=True)
    model.save_pretrained(out_dir)
    tokenizer.save_pretrained(out_dir)
    print(f"Saved flat baseline model to {out_dir}")

if __name__ == "__main__":
    main()


    AutoModelForSequenceClassification,
    AutoTokenizer,
    PreTrainedTokenizerBase,
)
import os

from tolbert.config import load_tolbert_config

class FlatLeafDataset(Dataset):
    """
    Minimal dataset for flat (leaf-level) classification.

    Expects the same JSONL format as TreeOfLifeDataset, but uses only:
      - "text": span text
      - "node_path": [root_id, ..., leaf_id]

    The label is taken to be the *last* element of node_path (leaf).
    """

    def __init__(
        self,
        spans_file: str,
        tokenizer: PreTrainedTokenizerBase,
        max_length: int = 256,
    ) -> None:
        self.spans_file = spans_file
        self.tokenizer = tokenizer
        self.max_length = max_length

        self._records: List[Dict[str, Any]] = []
        with open(spans_file, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                obj = json.loads(line)
                if "text" not in obj or "node_path" not in obj:
                    # Skip spans that do not have both text and a path.
                    continue
                if not obj["node_path"]:
                    continue
                self._records.append(obj)

    def __len__(self) -> int:
        return len(self._records)

    def _tokenize(self, text: str) -> Dict[str, torch.Tensor]:
        enc = self.tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=self.max_length,
        )
        return {k: v.squeeze(0) for k, v in enc.items()}

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        rec = self._records[idx]
        tokens = self._tokenize(rec["text"])
        input_ids = tokens["input_ids"]
        attention_mask = tokens.get("attention_mask", torch.ones_like(input_ids))
        # Leaf = last element of node_path


        max_length: int = 256,
    ) -> None:
        self.spans_file = spans_file
        self.tokenizer = tokenizer
        self.max_length = max_length

        self._records: List[Dict[str, Any]] = []
        with open(spans_file, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                obj = json.loads(line)
                if "text" not in obj or "node_path" not in obj:
                    # Skip spans that do not have both text and a path.
                    continue
                if not obj["node_path"]:
                    continue
                self._records.append(obj)

    def __len__(self) -> int:
        return len(self._records)

    def _tokenize(self, text: str) -> Dict[str, torch.Tensor]:
        enc = self.tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=self.max_length,
        )
        return {k: v.squeeze(0) for k, v in enc.items()}

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        rec = self._records[idx]
        tokens = self._tokenize(rec["text"])
        input_ids = tokens["input_ids"]
        attention_mask = tokens.get("attention_mask", torch.ones_like(input_ids))
        # Leaf = last element of node_path
        node_path: List[int] = rec["node_path"]
        label = int(node_path[-1])
        return {


        self.tokenizer = tokenizer
        self.max_length = max_length

        self._records: List[Dict[str, Any]] = []
        with open(spans_file, "r", encoding="utf-8") as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                obj = json.loads(line)
                if "text" not in obj or "node_path" not in obj:
                    # Skip spans that do not have both text and a path.
                    continue
                if not obj["node_path"]:
                    continue
                self._records.append(obj)

    def __len__(self) -> int:
        return len(self._records)

    def _tokenize(self, text: str) -> Dict[str, torch.Tensor]:
        enc = self.tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=self.max_length,
        )
        return {k: v.squeeze(0) for k, v in enc.items()}

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        rec = self._records[idx]
        tokens = self._tokenize(rec["text"])
        input_ids = tokens["input_ids"]
        attention_mask = tokens.get("attention_mask", torch.ones_like(input_ids))
        # Leaf = last element of node_path
        node_path: List[int] = rec["node_path"]
        label = int(node_path[-1])
        return {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "labels": torch.tensor(label, dtype=torch.long),
        }

def collate_flat_batch(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
    input_ids = torch.stack([b["input_ids"] for b in batch], dim=0)
    attention_mask = torch.stack([b["attention_mask"] for b in batch], dim=0)
    labels = torch.stack([b["labels"] for b in batch], dim=0)


                if "text" not in obj or "node_path" not in obj:
                    # Skip spans that do not have both text and a path.
                    continue
                if not obj["node_path"]:
                    continue
                self._records.append(obj)

    def __len__(self) -> int:
        return len(self._records)

    def _tokenize(self, text: str) -> Dict[str, torch.Tensor]:
        enc = self.tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=self.max_length,
        )
        return {k: v.squeeze(0) for k, v in enc.items()}

    def __getitem__(self, idx: int) -> Dict[str, Any]:
        rec = self._records[idx]
        tokens = self._tokenize(rec["text"])
        input_ids = tokens["input_ids"]
        attention_mask = tokens.get("attention_mask", torch.ones_like(input_ids))
        # Leaf = last element of node_path
        node_path: List[int] = rec["node_path"]
        label = int(node_path[-1])
        return {
            "input_ids": input_ids,
            "attention_mask": attention_mask,
            "labels": torch.tensor(label, dtype=torch.long),
        }

def collate_flat_batch(batch: List[Dict[str, Any]]) -> Dict[str, Any]:
    input_ids = torch.stack([b["input_ids"] for b in batch], dim=0)
    attention_mask = torch.stack([b["attention_mask"] for b in batch], dim=0)
    labels = torch.stack([b["labels"] for b in batch], dim=0)
    return {
        "input_ids": input_ids,
        "attention_mask": attention_mask,
        "labels": labels,
    }

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="Train a flat (leaf-level) baseline classifier.")
    ap.add_argument(
        "--config",
        type=str,
        required=True,
        help="YAML/JSON config with at least spans_file, level_sizes, batch_size, num_epochs, lr.",


"""
Simple retrieval sandbox for TOLBERT.

This script:
  - loads a trained TOLBERT checkpoint and tokenizer,
  - encodes spans from a `spans_file`,
  - builds an in-memory index of embeddings,
  - lets you run ad-hoc queries over that index from the command line.

It is intended for small-scale experimentation and debugging, not production.
"""

import argparse
from pathlib import Path
from typing import Any, Dict, List, Tuple

import torch
from torch.nn import functional as F
from transformers import AutoTokenizer
import os

from tolbert.config import load_tolbert_config
from tolbert.data import TreeOfLifeDataset
from tolbert.modeling import TOLBERT, TOLBERTConfig

def build_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def encode_all_spans(
    model: TOLBERT,
    tokenizer,
    spans_file: str,
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Encode all spans in `spans_file` into a single tensor of embeddings,
    plus a parallel list of raw records.
    """
    dataset = TreeOfLifeDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=max_length,
    )

    embs: List[torch.Tensor] = []
    metas: List[Dict[str, Any]] = []

    for rec in dataset._records:  # type: ignore[attr-defined]
        # Directly reuse dataset internals to avoid extra tokenization logic.
        tokens = tokenizer(
            rec.text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        input_ids = tokens["input_ids"].to(device)
        attention_mask = tokens["attention_mask"].to(device)
        with torch.no_grad():
            out = model(input_ids=input_ids, attention_mask=attention_mask)
        embs.append(out["proj"].squeeze(0).cpu())

        metas.append(rec.raw)

    emb_mat = torch.stack(embs, dim=0) if embs else torch.empty(0)
    return emb_mat, metas

def retrieve(
    query_emb: torch.Tensor,
    index_embs: torch.Tensor,
    k: int = 5,
) -> List[int]:
    """
    Return indices of top-k most similar embeddings using cosine similarity.
    """
    if index_embs.numel() == 0:
        return []
    sims = F.cosine_similarity(query_emb.unsqueeze(0), index_embs, dim=1)
    topk = torch.topk(sims, k=min(k, sims.numel()))
    return topk.indices.tolist()

def interactive_loop(
    model: TOLBERT,
    tokenizer,
    index_embs: torch.Tensor,
    metas: List[Dict[str, Any]],
    max_length: int,
    device: torch.device,
) -> None:
    print("Entering interactive retrieval loop. Type 'exit' to quit.")
    while True:
        text = input("Query> ")
        if not text or text.strip().lower() in {"exit", "quit"}:
            break
        tokens = tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        with torch.no_grad():
            out = model(
                input_ids=tokens["input_ids"].to(device),
                attention_mask=tokens["attention_mask"].to(device),
            )
        q_emb = out["proj"].squeeze(0).cpu()
        idxs = retrieve(q_emb, index_embs, k=5)
        for i in idxs:
            meta = metas[i]
            snippet = meta.get("text", "")[:200].replace("\n", " ")
            print(f"- span_id={meta.get('span_id')} text={snippet!r}")

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="TOLBERT retrieval sandbox.")
    ap.add_argument("--config", type=str, required=True, help="Path to training config.")
    ap.add_argument("--checkpoint", type=str, required=True, help="Path to model .pt file.")
    ap.add_argument(
        "--spans_file",
        type=str,
        required=True,
        help="Spans JSONL file to index (e.g., code spans or paper spans).",
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)
    device = torch.device(args.device)

    tokenizer = AutoTokenizer.from_pretrained(
        cfg["base_model_name"],
        cache_dir="/data/checkpoints/",  # noqa: E501
    )

    checkpoint_path = Path(args.checkpoint)
    if not checkpoint_path.exists():
        raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}")

    model = build_model(cfg, str(checkpoint_path), device=device)

    print("Encoding index spans...")
    index_embs, metas = encode_all_spans(
        model=model,
        tokenizer=tokenizer,
        spans_file=args.spans_file,
        max_length=cfg.get("max_length", 256),
        device=device,
    )
    print(f"Indexed {index_embs.size(0)} spans.")

    interactive_loop(
        model=model,
        tokenizer=tokenizer,
        index_embs=index_embs,
        metas=metas,
        max_length=cfg.get("max_length", 256),
        device=device,
    )

if __name__ == "__main__":
    main()


  - builds an in-memory index of embeddings,
  - lets you run ad-hoc queries over that index from the command line.

It is intended for small-scale experimentation and debugging, not production.
"""

import argparse
from pathlib import Path
from typing import Any, Dict, List, Tuple

import torch
from torch.nn import functional as F
from transformers import AutoTokenizer
import os

from tolbert.config import load_tolbert_config
from tolbert.data import TreeOfLifeDataset
from tolbert.modeling import TOLBERT, TOLBERTConfig

def build_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def encode_all_spans(
    model: TOLBERT,
    tokenizer,
    spans_file: str,
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Encode all spans in `spans_file` into a single tensor of embeddings,
    plus a parallel list of raw records.
    """
    dataset = TreeOfLifeDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=max_length,
    )

    embs: List[torch.Tensor] = []


from tolbert.config import load_tolbert_config
from tolbert.data import TreeOfLifeDataset
from tolbert.modeling import TOLBERT, TOLBERTConfig

def build_model(cfg: Dict[str, Any], checkpoint: str, device: torch.device) -> TOLBERT:
    model_cfg = TOLBERTConfig(
        base_model_name=cfg["base_model_name"],
        level_sizes=cfg["level_sizes"],
        proj_dim=cfg.get("proj_dim", 256),
    )
    model = TOLBERT(model_cfg)
    state = torch.load(checkpoint, map_location="cpu")
    model.load_state_dict(state)
    model.to(device)
    model.eval()
    return model

def encode_all_spans(
    model: TOLBERT,
    tokenizer,
    spans_file: str,
    max_length: int,
    device: torch.device,
) -> Tuple[torch.Tensor, List[Dict[str, Any]]]:
    """
    Encode all spans in `spans_file` into a single tensor of embeddings,
    plus a parallel list of raw records.
    """
    dataset = TreeOfLifeDataset(
        spans_file=spans_file,
        tokenizer=tokenizer,
        max_length=max_length,
    )

    embs: List[torch.Tensor] = []
    metas: List[Dict[str, Any]] = []

    for rec in dataset._records:  # type: ignore[attr-defined]
        # Directly reuse dataset internals to avoid extra tokenization logic.
        tokens = tokenizer(
            rec.text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        input_ids = tokens["input_ids"].to(device)
        attention_mask = tokens["attention_mask"].to(device)
        with torch.no_grad():
            out = model(input_ids=input_ids, attention_mask=attention_mask)
        embs.append(out["proj"].squeeze(0).cpu())

        metas.append(rec.raw)

    emb_mat = torch.stack(embs, dim=0) if embs else torch.empty(0)
    return emb_mat, metas

def retrieve(
    query_emb: torch.Tensor,
    index_embs: torch.Tensor,
    k: int = 5,
) -> List[int]:
    """
    Return indices of top-k most similar embeddings using cosine similarity.
    """
    if index_embs.numel() == 0:
        return []
    sims = F.cosine_similarity(query_emb.unsqueeze(0), index_embs, dim=1)
    topk = torch.topk(sims, k=min(k, sims.numel()))
    return topk.indices.tolist()

def interactive_loop(
    model: TOLBERT,
    tokenizer,


retrieve

        # Directly reuse dataset internals to avoid extra tokenization logic.
        tokens = tokenizer(
            rec.text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        input_ids = tokens["input_ids"].to(device)
        attention_mask = tokens["attention_mask"].to(device)
        with torch.no_grad():
            out = model(input_ids=input_ids, attention_mask=attention_mask)
        embs.append(out["proj"].squeeze(0).cpu())

        metas.append(rec.raw)

    emb_mat = torch.stack(embs, dim=0) if embs else torch.empty(0)
    return emb_mat, metas

def retrieve(
    query_emb: torch.Tensor,
    index_embs: torch.Tensor,
    k: int = 5,
) -> List[int]:
    """
    Return indices of top-k most similar embeddings using cosine similarity.
    """
    if index_embs.numel() == 0:
        return []
    sims = F.cosine_similarity(query_emb.unsqueeze(0), index_embs, dim=1)
    topk = torch.topk(sims, k=min(k, sims.numel()))
    return topk.indices.tolist()

def interactive_loop(
    model: TOLBERT,
    tokenizer,
    index_embs: torch.Tensor,
    metas: List[Dict[str, Any]],
    max_length: int,
    device: torch.device,
) -> None:
    print("Entering interactive retrieval loop. Type 'exit' to quit.")
    while True:
        text = input("Query> ")
        if not text or text.strip().lower() in {"exit", "quit"}:
            break
        tokens = tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",


    emb_mat = torch.stack(embs, dim=0) if embs else torch.empty(0)
    return emb_mat, metas

def retrieve(
    query_emb: torch.Tensor,
    index_embs: torch.Tensor,
    k: int = 5,
) -> List[int]:
    """
    Return indices of top-k most similar embeddings using cosine similarity.
    """
    if index_embs.numel() == 0:
        return []
    sims = F.cosine_similarity(query_emb.unsqueeze(0), index_embs, dim=1)
    topk = torch.topk(sims, k=min(k, sims.numel()))
    return topk.indices.tolist()

def interactive_loop(
    model: TOLBERT,
    tokenizer,
    index_embs: torch.Tensor,
    metas: List[Dict[str, Any]],
    max_length: int,
    device: torch.device,
) -> None:
    print("Entering interactive retrieval loop. Type 'exit' to quit.")
    while True:
        text = input("Query> ")
        if not text or text.strip().lower() in {"exit", "quit"}:
            break
        tokens = tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        with torch.no_grad():
            out = model(
                input_ids=tokens["input_ids"].to(device),
                attention_mask=tokens["attention_mask"].to(device),
            )
        q_emb = out["proj"].squeeze(0).cpu()
        idxs = retrieve(q_emb, index_embs, k=5)
        for i in idxs:
            meta = metas[i]
            snippet = meta.get("text", "")[:200].replace("\n", " ")
            print(f"- span_id={meta.get('span_id')} text={snippet!r}")

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="TOLBERT retrieval sandbox.")
    ap.add_argument("--config", type=str, required=True, help="Path to training config.")
    ap.add_argument("--checkpoint", type=str, required=True, help="Path to model .pt file.")
    ap.add_argument(
        "--spans_file",
        type=str,
        required=True,
        help="Spans JSONL file to index (e.g., code spans or paper spans).",
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    return ap.parse_args()


        tokens = tokenizer(
            text,
            return_tensors="pt",
            truncation=True,
            padding="max_length",
            max_length=max_length,
        )
        with torch.no_grad():
            out = model(
                input_ids=tokens["input_ids"].to(device),
                attention_mask=tokens["attention_mask"].to(device),
            )
        q_emb = out["proj"].squeeze(0).cpu()
        idxs = retrieve(q_emb, index_embs, k=5)
        for i in idxs:
            meta = metas[i]
            snippet = meta.get("text", "")[:200].replace("\n", " ")
            print(f"- span_id={meta.get('span_id')} text={snippet!r}")

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="TOLBERT retrieval sandbox.")
    ap.add_argument("--config", type=str, required=True, help="Path to training config.")
    ap.add_argument("--checkpoint", type=str, required=True, help="Path to model .pt file.")
    ap.add_argument(
        "--spans_file",
        type=str,
        required=True,
        help="Spans JSONL file to index (e.g., code spans or paper spans).",
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)
    device = torch.device(args.device)

    tokenizer = AutoTokenizer.from_pretrained(
        cfg["base_model_name"],
        cache_dir="/data/checkpoints/",  # noqa: E501
    )

    checkpoint_path = Path(args.checkpoint)
    if not checkpoint_path.exists():
        raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}")

    model = build_model(cfg, str(checkpoint_path), device=device)

    print("Encoding index spans...")
    index_embs, metas = encode_all_spans(


main

def parse_args() -> argparse.Namespace:
    ap = argparse.ArgumentParser(description="TOLBERT retrieval sandbox.")
    ap.add_argument("--config", type=str, required=True, help="Path to training config.")
    ap.add_argument("--checkpoint", type=str, required=True, help="Path to model .pt file.")
    ap.add_argument(
        "--spans_file",
        type=str,
        required=True,
        help="Spans JSONL file to index (e.g., code spans or paper spans).",
    )
    ap.add_argument(
        "--device",
        type=str,
        default="cuda" if torch.cuda.is_available() else "cpu",
        help="Device to use.",
    )
    return ap.parse_args()

def main() -> None:
    args = parse_args()
    cfg = load_tolbert_config(args.config)
    device = torch.device(args.device)

    tokenizer = AutoTokenizer.from_pretrained(
        cfg["base_model_name"],
        cache_dir="/data/checkpoints/",  # noqa: E501
    )

    checkpoint_path = Path(args.checkpoint)
    if not checkpoint_path.exists():
        raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}")

    model = build_model(cfg, str(checkpoint_path), device=device)

    print("Encoding index spans...")
    index_embs, metas = encode_all_spans(
        model=model,
        tokenizer=tokenizer,
        spans_file=args.spans_file,
        max_length=cfg.get("max_length", 256),
        device=device,
    )
    print(f"Indexed {index_embs.size(0)} spans.")

    interactive_loop(
        model=model,
        tokenizer=tokenizer,
        index_embs=index_embs,
        metas=metas,
        max_length=cfg.get("max_length", 256),
        device=device,
    )

if __name__ == "__main__":
    main()


import os
import ast
import re
import json
import time  # noqa: F401
from dataclasses import dataclass, field
from typing import Any, Dict, List, Tuple, Optional
try:
    import pathspec  # type: ignore
except Exception:  # pragma: no cover
    pathspec = None  # type: ignore

@dataclass
class Symbol:
    fqn: str
    name: str
    qualname: str
    kind: str  # module|class|function|variable
    module: str
    file: str
    line: int
    end_line: int
    doc: Optional[str] = None
    signature: Optional[str] = None
    returns: Optional[str] = None

@dataclass
class ModuleInfo:
    module: str
    file: str
    is_test: bool = False
    imports: Dict[str, str] = field(
        default_factory=dict
    )  # alias -> target (module or module.symbol)
    defs: List[str] = field(default_factory=list)  # list of symbol FQNs
    exports: List[str] = field(default_factory=list)  # names from __all__

class CodeGraph:
    def __init__(self, root: str, *, ignore: Optional[List[str]] = None) -> None:
        self.root = os.path.abspath(root)
        # Ignore patterns (relative to root) or glob-like; simple prefix/glob matching
        self._ignore: List[str] = []
        if ignore:
            # normalize to forward-slash relative prefixes for matching
            for pat in ignore:
                if not pat:
                    continue
                p = os.path.normpath(pat)
                # store both relative and absolute forms for convenience
                self._ignore.append(p)
        # Load .gitignore as pathspec if available
        self._pspec = None
        try:
            gi = os.path.join(self.root, ".gitignore")
            if pathspec is not None and os.path.exists(gi):
                with open(gi, "r", encoding="utf-8", errors="ignore") as fh:
                    lines = [ln.rstrip("\n") for ln in fh]
                self._pspec = pathspec.PathSpec.from_lines("gitwildmatch", lines)
        except Exception:
            self._pspec = None
        self.symbols_by_fqn: Dict[str, Symbol] = {}
        self.symbols_by_name: Dict[str, List[str]] = {}
        self.modules: Dict[str, ModuleInfo] = {}
        self.indexed_files: List[str] = []
        self.calls: List[Tuple[str, str]] = []  # (caller_fqn, callee_fqn_or_key)
        self.module_to_tests: Dict[str, List[str]] = {}
        self.coverage_files: Dict[str, set[int]] = {}
        self.symbol_coverage: Dict[str, float] = {}
        self.module_imports: Dict[str, List[str]] = {}
        self.module_star_imports: Dict[str, List[str]] = {}
        self.pytest_nodes_by_module: Dict[str, List[str]] = {}
        self._cached_mtimes: Dict[str, int] = {}
        self._cached_hashes: Dict[str, str] = {}

    def _is_ignored(self, rel: str) -> bool:
        try:
            r = rel.replace(os.sep, "/")
            # pathspec first
            if self._pspec is not None:
                if self._pspec.match_file(r):
                    return True
            # fallback: prefix match
            for pat in self._ignore:
                pp = pat.replace(os.sep, "/")
                if r == pp or r.startswith(pp + "/"):
                    return True
            return False
        except Exception:
            return False

    @classmethod
    def load_or_build(cls, root: str, *, ignore_cache: bool = False, ignore: Optional[List[str]] = None) -> "CodeGraph":
        g = cls(root=root, ignore=ignore)
        g.build(ignore_cache=ignore_cache)
        return g

    def build(self, ignore_cache: bool = False) -> None:
        cache_path = os.path.join(self.root, ".codegraph.json")
        if (not ignore_cache) and self._load_cache_relaxed(cache_path):
            # Incremental: reindex changed and dependents
            changed, removed = self._detect_changed_files(
                self._cached_mtimes, self._cached_hashes
            )
            if not changed and not removed:
                return
            self._incremental_reindex(changed, removed)
            self._expand_star_imports()
            self._post_resolve_calls()
            self._save_cache(cache_path)
            return
        for dirpath, dirnames, filenames in os.walk(self.root):
            # prune ignored directories in-place
            dir_rel = os.path.relpath(dirpath, self.root)
            # remove child dirs that are ignored
            dirnames[:] = [d for d in dirnames if not self._is_ignored(os.path.join(dir_rel, d))]
            if self._is_ignored(dir_rel):
                continue
            for fn in filenames:
                if not fn.endswith(".py"):
                    continue
                fpath = os.path.join(dirpath, fn)
                if self._is_ignored(os.path.relpath(fpath, self.root)):
                    continue
                try:
                    src = open(fpath, "r", encoding="utf-8").read()
                except Exception:
                    continue
                try:
                    tree = ast.parse(src)
                except Exception:
                    continue
                self.indexed_files.append(fpath)
                self._index_module(fpath, tree)
        # Build test mapping from imports in test modules
        self._build_test_mapping()
        # Expand star imports and post-resolve call targets
        self._expand_star_imports()
        self._post_resolve_calls()
        self._save_cache(cache_path)

    def _add_symbol(self, sym: Symbol) -> None:
        self.symbols_by_fqn[sym.fqn] = sym
        self.symbols_by_name.setdefault(sym.name, []).append(sym.fqn)
        mi = self.modules.setdefault(
            sym.module, ModuleInfo(module=sym.module, file=sym.file)
        )
        if sym.fqn not in mi.defs:
            mi.defs.append(sym.fqn)

    def _index_module(self, path: str, tree: ast.AST) -> None:
        module = self._module_name_for_path(path)
        is_test = ("/tests/" in path) or (os.path.basename(path).startswith("test_"))
        self.modules.setdefault(
            module, ModuleInfo(module=module, file=path, is_test=is_test)
        )
        # Add module symbol
        mod_fqn = module
        mod_name = module.split(".")[-1]
        self._add_symbol(
            Symbol(
                fqn=mod_fqn,
                name=mod_name,
                qualname="",
                kind="module",
                module=module,
                file=path,
                line=1,
                end_line=1,
            )
        )
        # Visit
        visitor = _ModuleVisitor(module, path)
        visitor.visit(tree)
        # Register imports
        self.modules[module].imports.update(visitor.imports)
        # Module dependency edges
        self.module_imports[module] = sorted(visitor.import_modules)
        # Record star imports for later expansion
        self.module_star_imports[module] = list(getattr(visitor, "star_imports", []))
        # Record __all__ exports
        self.modules[module].exports = list(getattr(visitor, "exports", []))
        # Register defs
        for sym in visitor.symbols:
            self._add_symbol(sym)
        # Register calls
        for caller, callee_key in visitor.calls:
            callee_fqn = self._resolve_callee(module, callee_key, visitor)
            self.calls.append((caller, callee_fqn or callee_key))
        # Collect pytest nodes if test module
        if is_test:
            rel = os.path.relpath(path, self.root)
            self.pytest_nodes_by_module[module] = self._collect_pytest_nodes(tree, rel)

    def owners_of(self, symbol: str) -> List[str]:
        fqns = self.symbols_by_name.get(symbol, [])
        return sorted(
            {os.path.relpath(self.symbols_by_fqn[f].file, self.root) for f in fqns}
        )

    def find_symbol(self, name: str) -> List[Symbol]:
        return [self.symbols_by_fqn[f] for f in self.symbols_by_name.get(name, [])]

    def defs_in(self, module: str) -> List[str]:
        mi = self.modules.get(module)
        return list(mi.defs) if mi else []

    def calls_of(self, fqn: str) -> List[str]:
        return [c for (caller, c) in self.calls if caller == fqn]

    def who_calls(self, fqn: str) -> List[str]:
        target_short = fqn.split(".")[-1]
        out: List[str] = []
        for caller, callee in self.calls:
            if callee == fqn or callee.split(".")[-1] == target_short:
                out.append(caller)
        return out

    def search_refs(self, pattern: str) -> List[Tuple[str, int, str]]:
        """Ripgrep-based raw reference search (file, line_no, text)."""
        try:
            import subprocess

            out = subprocess.check_output(["rg", "-n", pattern, self.root], text=True)
            rows: List[Tuple[str, int, str]] = []
            for line in out.splitlines():
                try:
                    fp, ln, txt = line.split(":", 2)
                    rows.append((os.path.relpath(fp, self.root), int(ln), txt))
                except Exception:
                    continue
            return rows
        except Exception:
            # Fallback: simple Python regex over indexed .py files
            rows: List[Tuple[str, int, str]] = []
            try:
                rx = re.compile(pattern)
            except Exception:
                # If pattern is not a valid regex, escape it
                rx = re.compile(re.escape(pattern))
            for fpath in self.indexed_files:
                rel = os.path.relpath(fpath, self.root)
                try:
                    with open(fpath, "r", encoding="utf-8", errors="ignore") as rf:
                        for i, ln in enumerate(rf, start=1):
                            if rx.search(ln):
                                rows.append((rel, i, ln.rstrip("\n")))
                except Exception:
                    continue
            return rows

    # --- Helpers --- #

    def module_for_file(self, path: str) -> Optional[str]:
        p = path
        if not os.path.isabs(p):
            p = os.path.abspath(os.path.join(self.root, path))
        for mod, mi in self.modules.items():
            if os.path.abspath(mi.file) == p:
                return mod
        return None

    def file_for_module(self, module: str) -> Optional[str]:
        mi = self.modules.get(module)
        return mi.file if mi else None

    def tests_for_module(self, module: str) -> List[str]:
        base = module.split(".")[0]
        out = set(self.module_to_tests.get(base, []))
        # include direct module key if present
        out.update(self.module_to_tests.get(module, []))
        return sorted(out)

    def tests_for_symbol(self, fqn: str) -> List[str]:
        mod = fqn.rsplit(".", 1)[0] if "." in fqn else fqn
        return self.tests_for_module(mod)

    def refs_of(self, fqn: str) -> List[Tuple[str, str]]:
        """Return (caller_fqn, callee_match) entries that reference fqn or its short name."""
        target_short = fqn.split(".")[-1]
        out: List[Tuple[str, str]] = []
        for caller, callee in self.calls:
            if callee == fqn or callee.split(".")[-1] == target_short:
                out.append((caller, callee))
        return out

    def export_json(self) -> Dict[str, Any]:
        return {
            "root": self.root,
            "files": [os.path.relpath(p, self.root) for p in self.indexed_files],
            "symbols": [self._sym_to_dict(s) for s in self.symbols_by_fqn.values()],
            "modules": {k: self._mi_to_dict(v) for k, v in self.modules.items()},
            "calls": self.calls,
            "module_to_tests": self.module_to_tests,
            "coverage_files": {
                os.path.relpath(k, self.root): sorted(list(v))
                for k, v in self.coverage_files.items()
            },
            "symbol_coverage": self.symbol_coverage,
            "module_imports": self.module_imports,
        }

    def export_sqlite(self, db_path: str) -> None:
        import sqlite3

        conn = sqlite3.connect(db_path)
        cur = conn.cursor()
        cur.executescript(
            """
            PRAGMA journal_mode=WAL;
            CREATE TABLE IF NOT EXISTS files(path TEXT PRIMARY KEY);
            CREATE TABLE IF NOT EXISTS modules(module TEXT PRIMARY KEY, file TEXT, is_test INT);
            CREATE TABLE IF NOT EXISTS symbols(
              fqn TEXT PRIMARY KEY, name TEXT, qualname TEXT, kind TEXT, module TEXT,
              file TEXT, line INT, end_line INT, doc TEXT, signature TEXT, returns TEXT
            );
            CREATE TABLE IF NOT EXISTS calls(caller TEXT, callee TEXT);
            CREATE TABLE IF NOT EXISTS tests_map(module TEXT, test_module TEXT);
            CREATE TABLE IF NOT EXISTS coverage(file TEXT, line INT);
            CREATE TABLE IF NOT EXISTS mod_deps(module TEXT, dep TEXT);
            """
        )
        cur.executemany(
            "INSERT OR IGNORE INTO files(path) VALUES(?)",
            [(os.path.relpath(f, self.root),) for f in self.indexed_files],
        )
        cur.executemany(
            "INSERT OR REPLACE INTO modules(module,file,is_test) VALUES(?,?,?)",
            [
                (m, os.path.relpath(mi.file, self.root), 1 if mi.is_test else 0)
                for m, mi in self.modules.items()
            ],
        )
        cur.executemany(
            "INSERT OR REPLACE INTO symbols VALUES(?,?,?,?,?,?,?,?,?,?,?)",
            [
                (
                    s.fqn,
                    s.name,
                    s.qualname,
                    s.kind,
                    s.module,
                    os.path.relpath(s.file, self.root),
                    int(s.line),
                    int(s.end_line),
                    s.doc or "",
                    s.signature or "",
                    s.returns or "",
                )
                for s in self.symbols_by_fqn.values()
            ],
        )
        if self.calls:
            cur.executemany(
                "INSERT INTO calls(caller,callee) VALUES(?,?)", list(self.calls)
            )
        rows = []
        for mod, tests in self.module_to_tests.items():
            for t in tests:
                rows.append((mod, t))
        if rows:
            cur.executemany(
                "INSERT INTO tests_map(module,test_module) VALUES(?,?)", rows
            )
        cov_rows = []
        for f, lines in self.coverage_files.items():
            rel = os.path.relpath(f, self.root)
            cov_rows.extend([(rel, int(n)) for n in lines])
        if cov_rows:
            cur.executemany("INSERT INTO coverage(file,line) VALUES(?,?)", cov_rows)
        dep_rows = []
        for m, deps in self.module_imports.items():
            for d in deps:
                dep_rows.append((m, d))
        if dep_rows:
            cur.executemany("INSERT INTO mod_deps(module,dep) VALUES(?,?)", dep_rows)
        conn.commit()
        conn.close()

    def _module_name_for_path(self, path: str) -> str:
        rel = os.path.relpath(path, self.root)
        no_ext = rel[:-3] if rel.endswith(".py") else rel
        parts = no_ext.split(os.sep)
        if parts[-1] == "__init__":
            parts = parts[:-1]
        return ".".join(p for p in parts if p)

    def _resolve_callee(
        self, module: str, callee_key: str, visitor: "_ModuleVisitor"
    ) -> Optional[str]:
        if "." in callee_key and ":" not in callee_key:
            return callee_key

        if ":" in callee_key:
            mod_alias, name = callee_key.split(":", 1)
            target = visitor.imports.get(mod_alias)
            if target:
                return f"{target}.{name}" if not target.endswith(f".{name}") else target

        mi = self.modules.get(module)
        if mi:
            # Prefer any def with same suffix name (matches within class or function)
            for f in mi.defs:
                if f.split(".")[-1] == callee_key:
                    return f

        tgt = visitor.imports.get(callee_key)
        if tgt:
            return tgt
        return None

    def _build_test_mapping(self) -> None:
        for mod, mi in self.modules.items():
            if not mi.is_test:
                continue
            for alias, target in mi.imports.items():
                # target may be module or module.symbol
                m = target.split(".")[0]
                self.module_to_tests.setdefault(m, []).append(mod)

    def _expand_star_imports(self) -> None:
        for mod, stars in self.module_star_imports.items():
            mi = self.modules.get(mod)
            if not mi:
                continue
            for star_mod in stars:
                defs = [
                    f
                    for f in self.modules.get(
                        star_mod, ModuleInfo(module=star_mod, file="")
                    ).defs
                ]
                exports = set(
                    self.modules.get(
                        star_mod, ModuleInfo(module=star_mod, file="")
                    ).exports
                    or []
                )
                for fqn in defs:
                    name = fqn.split(".")[-1]
                    if exports:
                        if name not in exports:
                            continue
                    elif name.startswith("_"):
                        continue
                    if name not in mi.imports:
                        mi.imports[name] = f"{star_mod}.{name}"

    def _post_resolve_calls(self) -> None:
        # After imports expanded, try to resolve unresolved simple names
        new_calls: List[Tuple[str, str]] = []
        for caller, callee in self.calls:
            if "." in callee:
                new_calls.append((caller, callee))
                continue
            # Find caller module
            caller_mod = caller.rsplit(".", 1)[0] if "." in caller else caller
            imports = self.modules.get(
                caller_mod, ModuleInfo(module=caller_mod, file="")
            ).imports
            tgt = imports.get(callee)
            if tgt:
                new_calls.append((caller, tgt))
            else:
                # leave as-is
                new_calls.append((caller, callee))
        self.calls = new_calls

    def unresolved_calls(self) -> List[Tuple[str, str]]:
        return [
            (a, c)
            for (a, c) in self.calls
            if "." not in c and not self._is_builtin_name(c)
        ]

    def _collect_pytest_nodes(self, tree: ast.AST, rel_path: str) -> List[str]:
        nodes: List[str] = []
        # top-level test_* functions
        for n in getattr(tree, "body", []) or []:
            if isinstance(n, ast.FunctionDef) and n.name.startswith("test_"):
                nodes.extend(self._expand_parametrize(rel_path, None, n))
            if isinstance(n, ast.ClassDef) and n.name.startswith("Test"):
                cls = n.name
                for m in getattr(n, "body", []) or []:
                    if isinstance(m, ast.FunctionDef) and m.name.startswith("test_"):
                        nodes.extend(self._expand_parametrize(rel_path, cls, m))
        return nodes

    def _expand_parametrize(
        self, rel_path: str, cls: Optional[str], fn: ast.FunctionDef
    ) -> List[str]:
        base = f"{rel_path}::" + (f"{cls}::" if cls else "") + fn.name
        # Look for @pytest.mark.parametrize("arg", [vals])
        total: List[str] = []
        params: List[int] = []
        try:
            for dec in getattr(fn, "decorator_list", []) or []:
                # pytest.mark.parametrize(...)
                if (
                    isinstance(dec, ast.Call)
                    and isinstance(dec.func, ast.Attribute)
                    and dec.func.attr == "parametrize"
                ):
                    # estimate number of cases from second arg list length
                    if len(dec.args) >= 2 and isinstance(
                        dec.args[1], (ast.List, ast.Tuple)
                    ):
                        params.append(len(dec.args[1].elts))
        except Ex
# ... truncated ...


Symbol

import os
import ast
import re
import json
import time  # noqa: F401
from dataclasses import dataclass, field
from typing import Any, Dict, List, Tuple, Optional
try:
    import pathspec  # type: ignore
except Exception:  # pragma: no cover
    pathspec = None  # type: ignore

@dataclass
class Symbol:
    fqn: str
    name: str
    qualname: str
    kind: str  # module|class|function|variable
    module: str
    file: str
    line: int
    end_line: int
    doc: Optional[str] = None
    signature: Optional[str] = None
    returns: Optional[str] = None

@dataclass
class ModuleInfo:
    module: str
    file: str
    is_test: bool = False
    imports: Dict[str, str] = field(
        default_factory=dict
    )  # alias -> target (module or module.symbol)
    defs: List[str] = field(default_factory=list)  # list of symbol FQNs
    exports: List[str] = field(default_factory=list)  # names from __all__

class CodeGraph:
    def __init__(self, root: str, *, ignore: Optional[List[str]] = None) -> None:
        self.root = os.path.abspath(root)
        # Ignore patterns (relative to root) or glob-like; simple prefix/glob matching
        self._ignore: List[str] = []
        if ignore: