scripts/build_publication_package.py

"""Build a deterministic compact publication package without raw trajectories."""

from __future__ import annotations

import gzip
import hashlib
import io
import json
import subprocess
import tarfile
from pathlib import Path


ROOT = Path(__file__).resolve().parents[1]
PACKAGE_NAME = "agent-harness-publication-package"
ROOT_FILES = (
    ".zenodo.json",
    "CITATION.cff",
    "LICENSE",
    "LICENSE-DATA",
    ".gitignore",
    "README.md",
    "README_ZENODO.md",
    "REPRODUCING.md",
    "THIRD_PARTY_NOTICES.md",
    "pyproject.toml",
    "requirements-analysis.txt",
    "requirements.lock",
)
SOURCE_DIRS = (
    "configs",
    "docs",
    "paper",
    "scripts",
    "src",
    "tasks",
    "tests",
    "output/pdf",
    "results/derived/confirmatory_analysis_174ce71bcbce",
    "results/derived/e07",
    "results/derived/study2",
    "results/derived/study3",
    "results/derived/study4",
    "results/derived/study4_ancillary",
    "results/derived/study5",
)
EXTRA_FILES = (
    "results/reports/study2_preflight.json",
    "results/reports/study3_preflight.json",
    "results/reports/study4_preflight.json",
    "results/reports/study4_ancillary_preflight.json",
    "results/reports/study5_preflight.json",
)


def sha256(data: bytes) -> str:
    return hashlib.sha256(data).hexdigest()


def collect_files(root: Path) -> list[Path]:
    candidates = [root / path for path in ROOT_FILES + EXTRA_FILES]
    for directory in SOURCE_DIRS:
        candidates.extend((root / directory).rglob("*"))
    files = sorted({path for path in candidates if path.is_file() and
                    "__pycache__" not in path.parts and path.suffix != ".pyc"})
    missing = [str(root / path) for path in ROOT_FILES + EXTRA_FILES
               if not (root / path).is_file()]
    if missing:
        raise FileNotFoundError(f"required publication files are missing: {missing}")
    return files


def tar_info(name: str, size: int) -> tarfile.TarInfo:
    info = tarfile.TarInfo(name=name)
    info.size = size
    info.mode = 0o644
    info.mtime = 0
    info.uid = info.gid = 0
    info.uname = info.gname = ""
    return info


def build(root: Path = ROOT) -> dict[str, object]:
    files = collect_files(root)
    revision = subprocess.run(
        ["git", "rev-parse", "HEAD"], cwd=root, check=True,
        capture_output=True, text=True,
    ).stdout.strip()
    entries = []
    payloads: list[tuple[str, bytes]] = []
    for path in files:
        relative = path.relative_to(root).as_posix()
        data = path.read_bytes()
        entries.append({"path": relative, "bytes": len(data), "sha256": sha256(data)})
        payloads.append((relative, data))

    package_metadata = {
        "schema_version": 1,
        "title": "Dissecting Repository-Scale Code-Agent Harnesses",
        "author": "Mandeep Sidhu",
        "version": "1.0.0-preprint",
        "doi": "10.5281/zenodo.21781711",
        "licenses": {"software": "MIT", "paper_and_derived_data": "CC-BY-4.0"},
        "source_revision": revision,
        "reported_cells": 5453,
        "reported_generative_tokens": 368125497,
        "study2_cells": 912,
        "study3_cells": 540,
        "study4_cells": 210,
        "study5_cells": 2826,
        "file_count": len(entries),
        "contents": "compact source, derived evidence, paper, and preflight",
        "excluded": ["raw trajectories", "repository checkouts", "index caches", "model weights"],
    }
    manifest = {
        "package": package_metadata,
        "files": entries,
    }
    metadata_bytes = json.dumps(package_metadata, indent=2, sort_keys=True).encode() + b"\n"
    manifest_bytes = json.dumps(manifest, indent=2, sort_keys=True).encode() + b"\n"

    output_dir = root / "output" / "releases"
    output_dir.mkdir(parents=True, exist_ok=True)
    archive = output_dir / f"{PACKAGE_NAME}.tar.gz"
    with archive.open("wb") as raw:
        with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed:
            with tarfile.open(fileobj=compressed, mode="w", format=tarfile.PAX_FORMAT) as tar:
                prefix = PACKAGE_NAME
                for name, data in payloads:
                    info = tar_info(f"{prefix}/{name}", len(data))
                    tar.addfile(info, io.BytesIO(data))
                for name, data in (
                    ("PUBLICATION_PACKAGE.json", metadata_bytes),
                    ("MANIFEST.sha256.json", manifest_bytes),
                ):
                    info = tar_info(f"{prefix}/{name}", len(data))
                    tar.addfile(info, io.BytesIO(data))

    archive_digest = sha256(archive.read_bytes())
    checksum = output_dir / f"{archive.name}.sha256"
    checksum.write_text(f"{archive_digest}  {archive.name}\n", encoding="utf-8")
    return {
        **package_metadata,
        "archive": str(archive),
        "archive_bytes": archive.stat().st_size,
        "archive_sha256": archive_digest,
        "checksum": str(checksum),
    }


def main() -> None:
    print(json.dumps(build(), indent=2, sort_keys=True))


if __name__ == "__main__":
    main()