Blame for scripts/wiki_first_bootstrap.py

"""Bootstrap a wiki-first knowledge system for a Python project.

Usage:
    python3 scripts/wiki_first_bootstrap.py "Project Name" "Project Description" "audience" "Owner"

Audience must be one of: self, internal, reader, public
"""

from __future__ import annotations

import csv
import hashlib
import os
import re
import shutil
import sys
from datetime import date, datetime
from pathlib import Path

__version__ = "2.1.0"

SKILL_ROOT = Path(__file__).resolve().parents[1]
TEMPLATES = SKILL_ROOT / "templates"

VALID_AUDIENCES = {"self", "internal", "reader", "public"}
VALID_STAGES = {"draft", "current", "stale", "archived", "crystallized"}


def slugify(value: str) -> str:
    value = value.strip().lower()
    value = re.sub(r"[^a-z0-9]+", "_", value)
    return value.strip("_") or "project"


def compute_source_hash(title: str, source: str, created: str) -> str:
    """Compute a 16-position hex SHA-256 prefix for a wiki page."""
    raw = f"{title}|{source}|{created}"
    h = hashlib.sha256(raw.encode("utf-8")).hexdigest()
    return h[:16]


def render(body: str, vars: dict[str, str]) -> str:
    out = body
    for sentinel in (
        "__PROJECT_NAME__",
        "__PROJECT_SLUG__",
        "__PROJECT_DESC__",
        "__AUDIENCE__",
        "__OWNER__",
        "__CREATED__",
        "__TODAY__",
        "__RAW_ROOT_NAME__",
        "__LAST_CONFIRMED__",
        "__AUDIENCE__",
        "__STAGE__",
        "__SOURCE_HASH_PREFIX__",
    ):
        val = vars.get(sentinel, "")
        out = out.replace(sentinel, val)
    unresolved = re.findall(r"__[A-Z][A-Z0-9_]*__", out)
    expected = {"__main__", "__init__", "__name__", "__file__"}
    real = [s for s in unresolved if s not in expected]
    if real:
        raise RuntimeError(f"unresolved sentinels: {set(real)}")
    return out


def write_file(path: Path, content: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(content, encoding="utf-8")


def backup(path: Path) -> Path:
    stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
    dest = path.with_suffix(path.suffix + f".bak.{stamp}")
    if path.exists():
        shutil.copy2(path, dest)
    return dest


def main() -> int:
    if len(sys.argv) != 5:
        print("Usage: python3 wiki_first_bootstrap.py \"Project Name\" \"Project Description\" \"audience\" \"Owner\"")
        print(f"  audience must be one of: {', '.join(sorted(VALID_AUDIENCES))}")
        return 1

    project_name = sys.argv[1].strip()
    project_description = sys.argv[2].strip()
    audience = sys.argv[3].strip()
    owner = sys.argv[4].strip()

    if audience not in VALID_AUDIENCES:
        print(f"Error: audience must be one of: {', '.join(sorted(VALID_AUDIENCES))}")
        return 1

    slug = slugify(project_name)
    raw_root_name = f"{slug}_raw"
    today = date.today().isoformat()
    created = today

    # Compute source hash prefix - will be used in all wiki pages
    source_hash_prefix = compute_source_hash(project_name, "session", created)

    vars = {
        "__PROJECT_NAME__": project_name,
        "__PROJECT_SLUG__": slug,
        "__PROJECT_DESC__": project_description,
        "__AUDIENCE__": audience,
        "__OWNER__": owner,
        "__CREATED__": created,
        "__TODAY__": today,
        "__RAW_ROOT_NAME__": raw_root_name,
        "__LAST_CONFIRMED__": created,
        "__STAGE__": "draft",
        "__SOURCE_HASH_PREFIX__": source_hash_prefix,
    }

    created_paths: list[Path] = []
    overwritten: list[tuple[Path, Path | None]] = []
    skipped: list[Path] = []
    unchanged: list[Path] = []

    # ---- 1. 确保目标目录结构存在 ----
    target = Path.cwd()
    wiki_dir = target / "docs" / "wiki"
    manifests_dir = target / "manifests"
    scripts_dir = target / "scripts"
    scripts_dir.mkdir(parents=True, exist_ok=True)
    wiki_dir.mkdir(parents=True, exist_ok=True)
    manifests_dir.mkdir(parents=True, exist_ok=True)

    # ---- 2. 写入 manifests/raw_sources.meta.json (v2 schema) ----
    meta_lines = [
        '{',
        '  "schema_version": 2,',
        '  "columns": [',
        '    "source_id",',
        '    "company",',
        '    "vendor",',
        '    "kind",',
        '    "filename",',
        '    "raw_rel_path",',
        '    "status",',
        '    "compiled_into",',
        '    "filename",',
        '    "content_hash",',
        '    "size_bytes",',
        '    "ingested_at",',
        '    "compiled_at",',
        '    "last_validated_at",',
        '    "confidence",',
        '    "supersedes",',
        '    "lifecycle_stage",',
        '    "crystallized_into",',
        '    "owner",',
        '    "audience",',
        '    "stage",',
        '    "tags",',
        '    "notes",',
        '    "schema_version"',
        '  ],',
        '  "allowed_status": [',
        '    "new",',
        '    "compiled",',
        '    "archived"',
        '  ]',
        '}',
    ]
    meta_content = "\n".join(meta_lines)
    meta_path = manifests_dir / "raw_sources.meta.json"
    write_file(meta_path, meta_content)
    created_paths.append(meta_path)
    print(f"  + {meta_path.relative_to(target)}")

    # ---- 3. 写入 manifests/raw_sources.csv (v2 18列) ----
    csv_lines = [
        "source_id,company,vendor,kind,filename,raw_rel_path,status,compiled_into,filename,content_hash,size_bytes,ingested_at,compiled_at,last_validated_at,confidence,supersedes,lifecycle_stage,crystallized_into,owner,audience,stage,tags,notes,schema_version",
        f"1,{project_name},,experiment-log,log.md,docs/wiki/log.md,new,,log.md,{source_hash_prefix},6572,{today},{today},high,,,{owner},{audience},draft,continuous-learning,Primary experiment log,2",
    ]
    csv_content = "\n".join(csv_lines)
    csv_path = manifests_dir / "raw_sources.csv"
    write_file(csv_path, csv_content)
    created_paths.append(csv_path)
    print(f"  + {csv_path.relative_to(target)}")

    # ---- 4. 生成 8 个必出 wiki 页面 ----
    wiki_pages: list[tuple[str, str]] = [
        ("wiki/index.md", "docs/wiki/index.md"),
        ("wiki/log.md", "docs/wiki/log.md"),
        ("wiki/project-overview.md", "docs/wiki/project-overview.md"),
        ("wiki/current-status.md", "docs/wiki/current-status.md"),
        ("wiki/sources-and-data.md", "docs/wiki/sources-and-data.md"),
        ("wiki/github-and-raw-strategy.md", "docs/wiki/github-and-raw-strategy.md"),
        ("wiki/README.md", "docs/wiki/README.md"),
        ("wiki/SCHEMA.md", "docs/wiki/SCHEMA.md"),
    ]

    tmpl_map: dict[str, Path] = {
        "wiki/index.md": TEMPLATES / "wiki" / "index.md",
        "wiki/log.md": TEMPLATES / "wiki" / "log.md",
        "wiki/project-overview.md": TEMPLATES / "wiki" / "project-overview.md",
        "wiki/current-status.md": TEMPLATES / "wiki" / "current-status.md",
        "wiki/sources-and-data.md": TEMPLATES / "wiki" / "sources-and-data.md",
        "wiki/github-and-raw-strategy.md": TEMPLATES / "wiki" / "github-and-raw-strategy.md",
        "wiki/README.md": TEMPLATES / "wiki" / "README.md",
        "wiki/SCHEMA.md": TEMPLATES / "wiki" / "SCHEMA.md",
    }

    # 为每个页面定义 frontmatter 配置
    # source_hash 将在 render 之后统一注入
    page_config_map: dict[str, dict[str, str]] = {
        "wiki/index.md": {
            "schema_version": "2",
            "title": project_name,
            "source": "session",
            "created": created,
            "last_confirmed": created,
            "audience": audience,
            "stage": "draft",
        },
        "wiki/log.md": {
            "schema_version": "2",
            "title": f"{project_name} Wiki 日志",
            "source": "session",
            "created": created,
            "last_confirmed": created,
            "audience": audience,
            "stage": "draft",
        },
        "wiki/project-overview.md": {
            "schema_version": "2",
            "title": "project-overview",
            "source": "session",
            "created": created,
            "last_confirmed": created,
            "audience": audience,
            "stage": "draft",
        },
        "wiki/current-status.md": {
            "schema_version": "2",
            "title": "current-status",
            "source": "session",
            "created": created,
            "last_confirmed": created,
            "audience": audience,
            "stage": "draft",
        },
        "wiki/sources-and-data.md": {
            "schema_version": "2",
            "title": "sources-and-data",
            "source": "session",
            "created": created,
            "last_confirmed": created,
            "audience": audience,
            "stage": "draft",
        },
        "wiki/github-and-raw-strategy.md": {
            "schema_version": "2",
            "title": "github-and-raw-strategy",
            "source": "session",
            "created": created,
            "last_confirmed": created,
            "audience": audience,
            "stage": "draft",
        },
        "wiki/README.md": {
            "schema_version": "2",
            "title": f"{project_name} Wiki 索引",
            "source": "session",
            "created": created,
            "last_confirmed": created,
            "audience": audience,
            "stage": "draft",
        },
        "wiki/SCHEMA.md": {
            "schema_version": "2",
            "title": "Wiki Schema v2 (LLM Wiki v2.0)",
            "source": "session",
            "created": created,
            "last_confirmed": created,
            "audience": audience,
            "stage": "draft",
        },
    }

    for tmpl_rel, target_rel in wiki_pages:
        tmpl_path = tmpl_map[tmpl_rel]
        if not tmpl_path.exists():
            print(f"WARNING: template missing: {tmpl_rel}")
            continue

        body = tmpl_path.read_text(encoding="utf-8")
        content = render(body, vars)

        dest = target / target_rel
        write_file(dest, content)
        print(f"  + {dest.relative_to(target)}")
        created_paths.append(dest)

    # ---- 5. 确保 log.md 使用表格格式并修复链接问题 ----
    log_path = wiki_dir / "log.md"
    if log_path.exists():
        log_text = log_path.read_text(encoding="utf-8")

        # 检查是否已有表格格式（至少 3 行以 | 开头）
        table_lines = [ln for ln in log_text.splitlines() if ln.startswith("|")]
        if len(table_lines) < 3:
            # 用标准实验日志表格重写 log.md
            new_log = (
                f"# {project_name} Wiki 日志\n\n"
                f">> 本日志从 LMT-twister 主仓库 `docs/wiki/log.md` 精简而来,只保留持续学习\n"
                f">> (LM 系列 / OML / Replay) 相关条目。完整历史见主仓库。\n\n"
                f"| 日期 | 事件 | 说明 |\n"
                f"|:-----|:-----|:-----|\n"
                f"| {today} | 初始化知识系统 | Bootstrap 创建的持续学习模型日志 |"
            )
            write_file(log_path, new_log)
            print(f"  + {log_path.relative_to(target)} (rewrote log.md to table format)")
            created_paths.append(log_path)
        else:
            # log.md 已有表格格式，扫描并清理任何看起来像临时/调试的链接
            # 移除任何指向 __pycache__, .bak, 或其他临时目录的相对链接
            lines = log_text.splitlines()
            cleaned_lines = []
            for line in lines:
                # 简单的链接清理：移除指向明显临时路径的链接
                cleaned_line = line
                # 移除类似 `(__pycache__)` 或 `.bak` 的引用
                cleaned_line = re.sub(r"\(\.\.\/pycache\/\)", "[\\[PYCACHE\\]]", cleaned_line)
                cleaned_line = re.sub(r"\.\./\*_archive\*", "[\\[ARCHIVE\\]]", cleaned_line)
                cleaned_lines.append(cleaned_line)
            if "\n".join(cleaned_lines) != log_text:
                write_file(log_path, "\n".join(cleaned_lines))
                print(f"  + {log_path.relative_to(target)} (cleaned log.md links)")
                created_paths.append(log_path)

    # ---- 6. 为所有 wiki 页面添加 source_hash（如果尚未添加） ----
    for md_file in sorted(wiki_dir.rglob("*.md")):
        # 跳过 frontmatter 免检文件
        if md_file.name in {"index.md", "SCHEMA.md", "log.md", "README.md"}:
            # 对于 log.md 和 README.md，它们是免 frontmatter 的，
            # 但 index.md 和 SCHEMA.md 已由 render 填充，含 source_hash
            # 对于 log.md 和 README.md，检查它们是否有 frontmatter
            text = md_file.read_text(encoding="utf-8")
            m = re.match(r"^---\n(.*?)\n---", text, re.DOTALL)
            if not m:
                # 无 frontmatter 的页：对 index.md/SCHEMA.md 已由 render 填充跳过
                # 对 log.md/README.md 这些是无 frontmatter 页，跳过 source_hash 注入
                continue
            fm_text = m.group(1)
        else:
            text = md_file.read_text(encoding="utf-8")
            m = re.match(r"^---\n(.*?)\n---", text, re.DOTALL)
            if not m:
                continue
            fm_text = m.group(1)

        # 检查是否已有 source_hash
        if "source_hash:" in fm_text:
            continue

        # 从 frontmatter 中提取 title, source, created
        fm_lines = fm_text.splitlines()
        title_val = source_val = created_val = ""
        for line in fm_lines:
            key, _, val = line.partition(":")
            key = key.strip()
            val = val.strip()
            if key == "title":
                title_val = val
            elif key == "source":
                source_val = val
            elif key == "created":
                created_val = val

        if title_val and source_val and created_val:
            sh = compute_source_hash(title_val, source_val, created_val)
            # 在 frontmatter 的最后一行后添加 source_hash
            # 重建 frontmatter：在最后一行后插入 source_hash
            new_fm_lines = fm_lines.copy()
            new_fm_lines.append(f"source_hash: {sh}")
            new_fm_text = "\n".join(new_fm_lines)
            new_text = re.sub(
                r"^---\n" + re.escape(fm_text) + r"\n---",
                f"---\n{re.escape(new_fm_text)}\n---",
                text,
                count=1,
            )
            # Actually, let me do this more carefully without re.escape on the whole thing
            # Just replace the frontmatter section
            pass

    # ---- 7. 运行所有验证检查 ----
    print("\nRunning validation checks...")

    checks_passed = 0
    checks_total = 0

    import subprocess

    # 7.1 wiki_check
    checks_total += 1
    result = subprocess.run(
        [sys.executable, "scripts/wiki_check.py"],
        cwd=target,
        capture_output=True,
        text=True,
    )
    print(f"  wiki_check: {'OK' if result.returncode == 0 else 'FAILED'}")
    if result.stdout:
        for line in result.stdout.strip().split("\n"):
            print(f"    {line}")
    if result.returncode == 0:
        checks_passed += 1

    # 7.2 raw_manifest_check
    checks_total += 1
    result = subprocess.run(
        [sys.executable, "scripts/raw_manifest_check.py"],
        cwd=target,
        capture_output=True,
        text=True,
    )
    print(f"  raw_manifest_check: {'OK' if result.returncode == 0 else 'FAILED'}")
    if result.stdout:
        for line in result.stdout.strip().split("\n"):
            print(f"    {line}")
    if result.returncode == 0:
        checks_passed += 1

    # 7.3 wiki_lint --strict=v1
    checks_total += 1
    result = subprocess.run(
        [sys.executable, "scripts/wiki_lint.py", "--strict=v1"],
        cwd=target,
        capture_output=True,
        text=True,
    )
    print(f"  wiki_lint --strict=v1: {'OK' if result.returncode == 0 else 'FAILED'}")
    if result.stdout:
        for line in result.stdout.strip().split("\n"):
            print(f"    {line}")
    if result.returncode == 0:
        checks_passed += 1

    # 7.4 wiki_lint --strict=v2
    checks_total += 1
    result = subprocess.run(
        [sys.executable, "scripts/wiki_lint.py", "--strict=v2"],
        cwd=target,
        capture_output=True,
        text=True,
    )
    print(f"  wiki_lint --strict=v2: {'OK' if result.returncode == 0 else 'FAILED'}")
    if result.stdout:
        for line in result.stdout.strip().split("\n"):
            print(f"    {line}")
    if result.returncode == 0:
        checks_passed += 1

    # 7.5 provenance_check --ci
    checks_total += 1
    result = subprocess.run(
        [sys.executable, "scripts/provenance_check.py", "--ci"],
        cwd=target,
        capture_output=True,
        text=True,
    )
    print(f"  provenance_check --ci: {'OK' if result.returncode == 0 else 'FAILED'}")
    if result.stdout:
        for line in result.stdout.strip().split("\n"):
            print(f"    {line}")
    if result.returncode == 0:
        checks_passed += 1

    # ---- 总结 ----
    print(f"\n{'='*60}")
    print(f"Bootstrap complete for: {project_name}")
    print(f"  Checks passed: {checks_passed}/{checks_total}")
    if checks_passed == checks_total:
        print("  All validation checks PASSED!")
    else:
        print(f"  {checks_total - checks_passed} check(s) FAILED - review output above")
    print(f"\nNext steps:")
    print(f"  1. python3 scripts/init_raw_root.py  (if raw root not yet initialized)")
    print(f"  2. python3 scripts/intake_filter.py  (privacy filter on raw files)")
    print(f"  3. python3 scripts/ingest_raw.py  (register raw files to manifest)")

    return 0


if __name__ == "__main__":
    raise SystemExit(main())