code/rebuild_skill.py

"""Rebuild a skill bundle (folder) from the bucket tables, byte-for-byte where content was stored.

  python code/rebuild_skill.py <sha | skill_uid> [--out DIR] [--with-children]

  sha        40-hex git blob SHA of a SKILL.md (the `sha` column of processed/canonical/skills): rebuilds the
             canonical bundle chosen for that content (canon_bundle_uid).
  skill_uid  a specific bundle: gh:{repo}:{skill_dir}@{commit12} (our crawl), gh:{repo}:{skill_dir} (GitSkills,
             Jul 2026), clawhub:{owner}/{slug}@{version}.

Writes DIR/<skill folder name>/SKILL.md + every bundle file at its rel_path, then checks each file against its git
blob SHA. Files listed without content (too large, binary > 256 KB, ...) are written to _MISSING.tsv with the reason.
--with-children also rebuilds nested skills under this folder (each child SKILL.md folder is its own bundle).

Lookup: processed/index/locator.parquet (skill_uid -> table file), built by build_locator.py.
"""
import argparse, hashlib, os, re, sys
from pathlib import Path, PurePosixPath
sys.path.insert(0, str(Path(__file__).resolve().parent))
from bucket_query import connect, BUCKET

BASE = f"https://huggingface.co/buckets/{BUCKET}/resolve/"
LOC = os.environ.get("SKILLS_LOCATOR", f"{BASE}processed/index/locator.parquet")  # or a local copy
c = connect("250MB")


def q(sql, *args): return c.execute(sql, list(args)).fetchall()


def git_sha(data): return hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest()


def locate(uid=None, prefix=None):
    """{table: [paths]} for one uid, or {uid: {table: [paths]}} for every uid starting with prefix."""
    if uid is not None:
        rows = q(f"select \"table\", path from read_parquet('{LOC}') where skill_uid = ?", uid)
        out = {}
        for t, p in rows: out.setdefault(t, []).append(p)
        return out
    rows = q(f"select skill_uid, \"table\", path from read_parquet('{LOC}') where starts_with(skill_uid, ?)", prefix)
    out = {}
    for u, t, p in rows: out.setdefault(u, {}).setdefault(t, []).append(p)
    return out


def paths_sql(ps): return "[" + ",".join(f"'{BASE}{p}'" for p in ps) + "]"


def rebuild(uid, root, rel_dir, report):
    where = locate(uid)
    if "skills" not in where: sys.exit(f"{uid}: not in the locator index")
    cur = c.execute(f"select * from read_parquet({paths_sql(where['skills'])}, union_by_name=true) "
                    f"where skill_uid = ? limit 1", [uid])
    row = dict(zip([d[0] for d in cur.description], cur.fetchone()))
    skill_md = row["skill_md"]
    files = []
    if where.get("files"):  # columns differ by source (ClawHub has no blob_sha / skipped): read by name
        cur = c.execute(f"select * from read_parquet({paths_sql(where['files'])}, union_by_name=true) where skill_uid = ?", [uid])
        cols = [d[0] for d in cur.description]
        files = [tuple(r.get(k) for k in ("rel_path", "size", "blob_sha", "content", "content_bin", "has_content", "skipped", "sha256"))
                 for r in (dict(zip(cols, v)) for v in cur.fetchall())]
    best = {}  # fill-pass rule: when a path has a placeholder row and a row with content, keep the one with content
    for f in files:
        if f[0] not in best or (f[5] and not best[f[0]][5]): best[f[0]] = f
    d = root / rel_dir; d.mkdir(parents=True, exist_ok=True)
    (d / "SKILL.md").write_bytes(skill_md.encode("utf-8"))
    missing, ok, bad = [], 0, []
    if row.get("file_sha"):
        if git_sha(skill_md.encode("utf-8")) == row["file_sha"]: ok += 1
        else: bad.append("SKILL.md")
    for rel, size, blob, text, binary, has, skipped, sha256 in sorted(best.values(), key=lambda f: f[0] or ""):
        if not has:
            missing.append(f"{rel}\t{size}\t{skipped}"); continue
        rp = PurePosixPath(rel)
        if rp.is_absolute() or ".." in rp.parts: missing.append(f"{rel}\t{size}\tunsafe_path"); continue
        data = binary if binary is not None else text.encode("utf-8")
        (d / rel).parent.mkdir(parents=True, exist_ok=True); (d / rel).write_bytes(data)
        if blob and re.fullmatch(r"[0-9a-f]{40}", blob):
            if git_sha(data) == blob: ok += 1
            else: bad.append(rel)
        elif sha256:  # ClawHub publishes sha256 per file
            if hashlib.sha256(data).hexdigest() == sha256: ok += 1
            else: bad.append(rel)
    if missing: (d / "_MISSING.tsv").write_text("rel_path\tsize\treason\n" + "\n".join(missing) + "\n")
    report.append((uid, rel_dir, 1 + len(best) - len(missing), len(missing), ok, bad))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("key"); ap.add_argument("--out", default="rebuilt"); ap.add_argument("--with-children", action="store_true")
    a = ap.parse_args()
    uid, key = a.key, a.key
    if re.fullmatch(r"[0-9a-f]{40}", key):
        r = q(f"select canon_bundle_uid, name from read_parquet('{BASE}processed/canonical/skills/skills-{key[0]}.parquet') "
              f"where sha = ?", key)
        if not r: sys.exit(f"sha {key} not in the canonical table")
        uid = r[0][0]; print(f"canonical bundle for {key[:12]} ({r[0][1]}): {uid}")
    m = re.fullmatch(r"gh:([^:]+):(.+?)(@[0-9a-f]{12})?", uid)
    name = (m.group(2).rstrip("/").rsplit("/", 1)[-1] if m and m.group(2) != "." else
            m.group(1).replace("/", "__") if m else uid.split(":", 1)[1].split("@")[0].replace("/", "__"))
    root, report = Path(a.out) / name, []
    rebuild(uid, root, ".", report)
    if a.with_children and m:
        repo, sd, at = m.group(1), m.group(2), m.group(3) or ""
        pre = f"gh:{repo}:" if sd == "." else f"gh:{repo}:{sd}/"
        for child in sorted(locate(prefix=pre)):
            cm = re.fullmatch(r"gh:[^:]+:(.+?)(@[0-9a-f]{12})?", child)
            if child == uid or not cm or (cm.group(2) or "") != at: continue  # same snapshot only
            rebuild(child, root, cm.group(1) if sd == "." else cm.group(1)[len(sd) + 1:], report)
    for u, rel, n, miss, ok, bad in report:
        print(f"{rel:40s} {n:4d} files written, {miss} listed-but-not-stored, {ok} verified (git blob SHA / sha256)"
              + (f", {len(bad)} MISMATCH: {bad[:3]}" if bad else "") + f"   [{u}]")
    print(f"-> {root}")


if __name__ == "__main__":
    main()