| """Create a fresh allowlisted release, never upload the whole working directory.""" |
| from pathlib import Path |
| from hashlib import sha256 |
| from datetime import datetime, timezone |
| import json |
| import shutil |
| import subprocess |
| import uuid |
|
|
| ROOT = Path(__file__).resolve().parents[1] |
| TOP_FILES = {"README.md", "LICENSE", "NOTICE", "pyproject.toml", "requirements-tested.txt", ".gitignore", ".gitattributes"} |
| SOURCE_DIRS = {"nexora", "scripts", "tests", "docs", "configs", "examples", "reports"} |
| ALLOWED_EXTENSIONS = {".py", ".md", ".json", ".jsonl", ".toml", ".txt", ".xml", ".png", ".svg", ".csv", ".ps1", ".sh"} |
|
|
|
|
| def main(): |
| |
| stage = ROOT / ".release" / ("stage-" + uuid.uuid4().hex[:12]) |
| stage.mkdir(parents=True) |
| files = [ROOT / name for name in TOP_FILES if (ROOT / name).is_file()] |
| for name in SOURCE_DIRS: |
| files.extend(p for p in (ROOT / name).rglob("*") if p.is_file() and p.suffix in ALLOWED_EXTENSIONS and "__pycache__" not in p.parts and not p.is_symlink()) |
| for folder in ["artifacts/data", "artifacts/tiny", "artifacts/posttraining-experiment"]: |
| files.extend(p for p in (ROOT / folder).glob("*") if p.is_file() and p.suffix in {".json", ".jsonl", ".npy", ".safetensors"}) |
| checkpoint_dir = ROOT / "artifacts/tiny/checkpoints" |
| latest = json.loads((checkpoint_dir / "latest.json").read_text()) |
| checkpoint = checkpoint_dir / latest["file"] |
| if checkpoint.resolve().parent != checkpoint_dir.resolve() or sha256(checkpoint.read_bytes()).hexdigest() != latest["sha256"]: |
| raise ValueError("Checkpoint manifest mismatch") |
| files.extend([checkpoint_dir / "latest.json", checkpoint]) |
| inventory = [] |
| for path in sorted(set(files)): |
| if path.is_symlink() or not path.resolve().is_relative_to(ROOT): |
| raise ValueError("Release path escapes project") |
| relative = path.relative_to(ROOT) |
| if relative.as_posix() in {"reports/release-manifest.json", "reports/upload-receipt.json"}: |
| continue |
| destination = stage / relative |
| destination.parent.mkdir(parents=True, exist_ok=True) |
| shutil.copy2(path, destination) |
| inventory.append({"path": relative.as_posix(), "bytes": destination.stat().st_size, |
| "sha256": sha256(destination.read_bytes()).hexdigest()}) |
| revision = subprocess.run(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip() |
| dirty = subprocess.run(["git", "status", "--porcelain"], cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip() |
| if dirty: |
| raise RuntimeError("Commit release inputs before packaging; source revision must identify the staged files") |
| manifest = {"created_utc": datetime.now(timezone.utc).isoformat(), "source_commit": revision, "repository": "devildasdf/NEXORA", |
| "original_model_parameters": 820736, "status": "research prototype, not trained 120B", "files": inventory, |
| "manifest_self_hash_excluded": True, "total_bytes": sum(x["bytes"] for x in inventory)} |
| (stage / "reports/release-manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") |
| (ROOT / ".release/current-stage.txt").write_text(str(stage), encoding="utf-8") |
| print(json.dumps({"stage": str(stage), "files": len(inventory), "bytes": manifest["total_bytes"], "source_commit": revision})) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
