File size: 5,392 Bytes
f40c7ba | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | #!/usr/bin/env python3
"""Push all BlitzKode artifacts to HuggingFace Hub in one command.
Uploads
-------
1. LoRA adapter (1.5B) → neuralbroker/blitzkode-1.5b-lora
2. LoRA adapter (0.5B) → neuralbroker/blitzkode-lora-0.5b
3. GGUF model file → neuralbroker/blitzkode (into a GGUF-specific branch/folder)
Usage
-----
# Export HF_TOKEN first, then run:
python scripts/push_all_to_hub.py
# Or pass token directly:
python scripts/push_all_to_hub.py --token hf_XXXX
# Dry-run to validate without pushing:
python scripts/push_all_to_hub.py --dry-run
"""
from __future__ import annotations
import argparse
import os
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[1]
PUSH_SCRIPT = REPO_ROOT / "scripts" / "push_to_hub.py"
ARTIFACTS = [
{
"label": "1.5B LoRA adapter (primary)",
"checkpoint": REPO_ROOT / "checkpoints" / "blitzkode-1.5b-lora" / "final",
"repo_id": "neuralbroker/blitzkode-1.5b-lora",
"commit_message": "Upload BlitzKode 1.5B LoRA adapter v2.1 (100-step SFT)",
},
{
"label": "0.5B LoRA adapter (lightweight)",
"checkpoint": REPO_ROOT / "checkpoints" / "available-lora-0.5b-full" / "final",
"repo_id": "neuralbroker/blitzkode-lora-0.5b",
"commit_message": "Upload BlitzKode 0.5B LoRA adapter v2.1 (50-step SFT)",
},
]
def push_gguf(token: str, gguf_path: Path, dry_run: bool) -> None:
if not gguf_path.exists():
print(f" [SKIP] GGUF not found: {gguf_path}")
return
size_gb = gguf_path.stat().st_size / 1024 ** 3
print(f"\n Uploading GGUF ({size_gb:.2f} GB) → neuralbroker/blitzkode ...")
if dry_run:
print(" [DRY RUN] skipped.")
return
from huggingface_hub import HfApi # noqa: PLC0415
from huggingface_hub.utils import HfHubHTTPError # noqa: PLC0415
api = HfApi(token=token)
try:
api.create_repo("neuralbroker/blitzkode", repo_type="model", exist_ok=True)
api.upload_file(
path_or_fileobj=str(gguf_path),
path_in_repo="blitzkode.gguf",
repo_id="neuralbroker/blitzkode",
repo_type="model",
commit_message="Update GGUF model Q8_0 (1.5B merged + quantised)",
)
print(" [OK] GGUF uploaded → https://huggingface.co/neuralbroker/blitzkode")
except HfHubHTTPError as exc:
print(f" [ERROR] GGUF upload failed: {exc}", file=sys.stderr)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--token", default=os.environ.get("HF_TOKEN", ""), help="HuggingFace write token (or set HF_TOKEN env var).")
parser.add_argument("--dry-run", action="store_true", help="Validate only, do not push.")
args = parser.parse_args()
token = args.token.strip()
if not token and not args.dry_run:
print(
"\n[ERROR] HuggingFace token required.\n"
" Option 1: export HF_TOKEN=hf_XXXX\n"
" Option 2: python scripts/push_all_to_hub.py --token hf_XXXX\n"
" Option 3: run --dry-run to validate without pushing\n"
"\nGet a write token at: https://huggingface.co/settings/tokens",
file=sys.stderr,
)
sys.exit(1)
print("=" * 72)
print("BLITZKODE — PUSH ALL ARTIFACTS TO HUGGING FACE HUB")
if args.dry_run:
print("(DRY RUN — nothing will be pushed)")
print("=" * 72)
failures: list[str] = []
for art in ARTIFACTS:
print(f"\n{'─' * 60}")
print(f" Artifact : {art['label']}")
print(f" Repo : {art['repo_id']}")
checkpoint: Path = art["checkpoint"]
if not checkpoint.exists():
print(f" [SKIP] Checkpoint not found: {checkpoint}")
continue
cmd = [
sys.executable,
str(PUSH_SCRIPT),
"--checkpoint",
str(checkpoint),
"--repo-id",
art["repo_id"],
"--commit-message",
art["commit_message"],
]
if args.dry_run:
cmd.append("--dry-run")
if token:
cmd += ["--token", token]
result = subprocess.run(cmd)
if result.returncode != 0:
failures.append(art["repo_id"])
print(f" [FAIL] Push exited with code {result.returncode}", file=sys.stderr)
# GGUF upload (direct via huggingface_hub)
print(f"\n{'─' * 60}")
print(" Artifact : GGUF model (neuralbroker/blitzkode)")
if not args.dry_run and token:
push_gguf(token, REPO_ROOT / "blitzkode.gguf", dry_run=False)
else:
push_gguf(token, REPO_ROOT / "blitzkode.gguf", dry_run=True)
# Summary
print(f"\n{'=' * 72}")
if failures:
print(f"PUSH FINISHED WITH FAILURES: {failures}")
sys.exit(1)
else:
print("ALL PUSHES COMPLETE")
print("\nHuggingFace repos:")
for art in ARTIFACTS:
print(f" https://huggingface.co/{art['repo_id']}")
print(" https://huggingface.co/neuralbroker/blitzkode (GGUF)")
if __name__ == "__main__":
main()
|