src/catalog_builder.py

#!/usr/bin/env python3
"""
catalog_builder.py -- Build a bulk skill catalog in the wiki.

Creates ~/.claude/skill-wiki/catalog.md listing ALL installed skills/agents
with name, path, line count, and category. This is the master index — individual
entity pages (entities/skills/*.md) are created on-demand by the router.

Usage:
    python catalog_builder.py \
      --wiki ~/.claude/skill-wiki \
      --skills-dir ~/.claude/skills \
      --agents-dir ~/.claude/agents \
      [--extra-dirs /path1 /path2]   # additional skill repos

Also adds newly found skills to the wiki index.md and appends to log.md.
"""

import argparse
import re
import sys
from datetime import datetime, timezone
from pathlib import Path

from ctx.core.wiki.wiki_packs import load_merged_wiki_pages, write_active_wiki_overlay_pack
from ctx_config import cfg

TODAY = datetime.now(timezone.utc).strftime("%Y-%m-%d")


def _read_wiki_page(wiki_dir: Path, relpath: str) -> str | None:
    """Read a wiki page from active packs when installed, else from disk."""
    packs_dir = wiki_dir / "wiki-packs"
    path = wiki_dir / relpath
    if packs_dir.is_dir():
        pages = load_merged_wiki_pages(packs_dir)
        if relpath in pages:
            return pages[relpath]
        if path.exists():
            return path.read_text(encoding="utf-8", errors="replace")
        return None
    if not path.exists():
        return None
    return path.read_text(encoding="utf-8", errors="replace")


def _write_wiki_page(wiki_dir: Path, relpath: str, content: str) -> None:
    """Write a wiki page, mirroring into overlay packs when installed."""
    packs_dir = wiki_dir / "wiki-packs"
    path = wiki_dir / relpath
    if path.exists() or not packs_dir.is_dir():
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(content, encoding="utf-8")
    if packs_dir.is_dir():
        write_active_wiki_overlay_pack(
            packs_dir=packs_dir,
            pages={relpath: content},
            tombstones=[],
        )


def scan_skills_dir(skills_dir: Path) -> list[dict]:
    """Scan a directory for skills (subdirs with SKILL.md)."""
    results: list[dict[str, object]] = []
    if not skills_dir.exists():
        return results

    for item in sorted(skills_dir.iterdir()):
        if item.is_dir():
            skill_md = item / "SKILL.md"
            if skill_md.exists():
                try:
                    lines = len(skill_md.read_text(encoding="utf-8", errors="replace").splitlines())
                except Exception as exc:
                    print(f"Warning: failed to read skill file {skill_md}: {exc}", file=sys.stderr)
                    lines = 0
                results.append({
                    "name": item.name,
                    "path": str(skill_md),
                    "lines": lines,
                    "type": "skill",
                    "over_180": lines > cfg.line_threshold,
                })
    return results


def scan_agents_dir(agents_dir: Path) -> list[dict]:
    """Scan a directory for flat agent .md files."""
    results: list[dict[str, object]] = []
    if not agents_dir.exists():
        return results

    for item in sorted(agents_dir.glob("*.md")):
        try:
            lines = len(item.read_text(encoding="utf-8", errors="replace").splitlines())
        except Exception as exc:
            print(f"Warning: failed to read agent file {item}: {exc}", file=sys.stderr)
            lines = 0
        results.append({
            "name": item.stem,
            "path": str(item),
            "lines": lines,
            "type": "agent",
            "over_180": lines > cfg.line_threshold,
        })
    return results


def build_catalog(
    wiki_dir: Path,
    skills_dir: Path,
    agents_dir: Path,
    extra_dirs: list[Path],
) -> dict:
    """Build catalog.md in the wiki and return stats."""
    all_items: list[dict] = []

    # Scan primary dirs
    all_items.extend(scan_skills_dir(skills_dir))
    all_items.extend(scan_agents_dir(agents_dir))

    # Scan extra dirs (additional skill repos)
    for extra in extra_dirs:
        if extra.is_dir():
            # Try both patterns: dir/SKILL.md and flat *.md
            sub_skills = scan_skills_dir(extra)
            if sub_skills:
                all_items.extend(sub_skills)
            else:
                all_items.extend(scan_agents_dir(extra))

    # Stats
    total = len(all_items)
    over_180 = sum(1 for i in all_items if i["over_180"])
    skills_count = sum(1 for i in all_items if i["type"] == "skill")
    agents_count = sum(1 for i in all_items if i["type"] == "agent")

    # Build catalog.md
    lines = [
        "# Skill Catalog",
        "",
        f"> Auto-generated by catalog_builder.py on {TODAY}.",
        "> Individual wiki pages are created on-demand when skills are loaded by the router.",
        "",
        "## Summary",
        "",
        "| Metric | Count |",
        "|--------|-------|",
        f"| Total items | {total} |",
        f"| Skills (SKILL.md) | {skills_count} |",
        f"| Agents (flat .md) | {agents_count} |",
        f"| Items > 180 lines | {over_180} |",
        f"| Items ≤ 180 lines | {total - over_180} |",
        "",
        "## All Skills",
        "",
        "| Name | Type | Lines | Over 180 | Path |",
        "|------|------|-------|----------|------|",
    ]

    for item in all_items:
        flag = "⚠" if item["over_180"] else ""
        lines.append(
            f"| {item['name']} | {item['type']} | {item['lines']} | {flag} | `{item['path']}` |"
        )

    catalog_path = wiki_dir / "catalog.md"
    _write_wiki_page(wiki_dir, "catalog.md", "\n".join(lines) + "\n")

    return {
        "total": total,
        "skills": skills_count,
        "agents": agents_count,
        "over_180": over_180,
        "catalog_path": str(catalog_path),
    }


def update_wiki_index(wiki_dir: Path, stats: dict) -> None:
    """Update index.md with catalog reference."""
    content = _read_wiki_page(wiki_dir, "index.md")
    if content is None:
        return
    catalog_ref = "- [[catalog]] - Full skill catalog (all installed items)"

    if "[[catalog]]" not in content:
        # Insert under ## Skills section
        lines = content.split("\n")
        for i, line in enumerate(lines):
            if line.strip() == "## Skills":
                lines.insert(i + 1, catalog_ref)
                break
        else:
            lines.append(catalog_ref)
        content = "\n".join(lines)

    # Update total count
    content = re.sub(
        r"Total pages: \d+",
        f"Total pages: {stats['total']}",
        content,
    )
    content = re.sub(
        r"Last updated: [\d-]+",
        f"Last updated: {TODAY}",
        content,
    )
    _write_wiki_page(wiki_dir, "index.md", content)


def append_log(wiki_dir: Path, stats: dict) -> None:
    """Append catalog build entry to log.md."""
    content = _read_wiki_page(wiki_dir, "log.md")
    if content is None:
        return

    entry = (
        f"\n## [{TODAY}] catalog-build | all-skills\n"
        f"- Total items cataloged: {stats['total']}\n"
        f"- Skills: {stats['skills']}, Agents: {stats['agents']}\n"
        f"- Over 180 lines (micro-skill candidates): {stats['over_180']}\n"
        f"- Catalog written to: {stats['catalog_path']}\n"
    )
    _write_wiki_page(wiki_dir, "log.md", content + entry)


def main() -> None:
    parser = argparse.ArgumentParser(description="Build bulk skill catalog in wiki")
    parser.add_argument("--wiki", default=str(cfg.wiki_dir), help="Wiki directory")
    parser.add_argument("--skills-dir", default=str(cfg.skills_dir), help="Skills directory")
    parser.add_argument("--agents-dir", default=str(cfg.agents_dir), help="Agents directory")
    parser.add_argument("--extra-dirs", nargs="*", default=[], help="Additional skill directories")
    args = parser.parse_args()

    wiki_dir = Path(args.wiki)
    if not wiki_dir.exists():
        print(f"Wiki not initialized at {wiki_dir}. Run wiki_sync.py --init first.", file=sys.stderr)
        sys.exit(1)

    stats = build_catalog(
        wiki_dir=wiki_dir,
        skills_dir=Path(args.skills_dir),
        agents_dir=Path(args.agents_dir),
        extra_dirs=[Path(d) for d in args.extra_dirs],
    )

    update_wiki_index(wiki_dir, stats)
    append_log(wiki_dir, stats)

    print(f"Catalog built: {stats['total']} items ({stats['over_180']} over 180 lines)")
    print(f"Written to: {stats['catalog_path']}")


if __name__ == "__main__":
    main()