Upload release.sh with huggingface_hub

release.sh

@@ -0,0 +1,136 @@
+#!/usr/bin/env bash
+# Publish a model artifact to the Enrell/lain-parser release channel.
+#
+#   bash release.sh <artifact-dir> <name> [--default] [--eval "note"]
+#   bash release.sh artifacts/e5-s2 e5-s2 --default \
+#       --eval "real-final-v1 int8 CPU: 137/196"
+#
+# Ordering is the no-break contract: model files land on the Hub first, the
+# manifest pointer moves last, so a manifest always references already-verified
+# content. Released names are immutable — a fix ships as a new name.
+#
+#   --snapshot <name>      pair the model with a snapshot already in the manifest
+#                          (default: the current default model's snapshot)
+#   --snapshot-dir <dir>   register + upload a NEW encoder snapshot (only needed
+#                          when the model pairs with a snapshot not yet released)
+#   --app                  also push install.sh, requirements.txt and app/ —
+#                          use whenever the vendored inference code changed
+#
+# Requires: hf CLI (authenticated, e.g. `hf auth login`), curl, python3.
+set -euo pipefail
+
+HF_REPO="${LAIN_PARSER_HF_REPO:-Enrell/lain-parser}"
+HF_BASE="https://huggingface.co/${HF_REPO}/resolve/main"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+step() { printf '\n== %s\n' "$*"; }
+die() { echo "error: $*" >&2; exit 1; }
+usage() { sed -n '3,18p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }
+
+ART_DIR="" NAME="" DEFAULT=0 EVAL="" SNAP_DIR="" SNAP_REF="" APP=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --default) DEFAULT=1; shift ;;
+    --eval) EVAL="$2"; shift 2 ;;
+    --snapshot) SNAP_REF="$2"; shift 2 ;;
+    --snapshot-dir) SNAP_DIR="$2"; shift 2 ;;
+    --app) APP=1; shift ;;
+    -h|--help) usage ;;
+    -*) echo "unknown argument: $1" >&2; exit 2 ;;
+    *) [[ -z "$ART_DIR" ]] && ART_DIR="$1" || NAME="$1"; shift ;;
+  esac
+done
+[[ -n "$ART_DIR" && -n "$NAME" ]] || usage
+[[ "$NAME" =~ ^[a-z0-9][a-z0-9.-]*$ ]] || { echo "bad model name: $NAME" >&2; exit 2; }
+[[ -f "${ART_DIR}/model.int8.pt" && -f "${ART_DIR}/config.json" ]] || {
+  echo "error: ${ART_DIR} needs model.int8.pt + config.json" >&2; exit 1; }
+
+command -v hf >/dev/null || die "hf CLI required (pip install huggingface_hub[cli], then hf auth login)"
+
+step "fetching current manifest"
+work="$(mktemp -d)"; trap 'rm -rf "$work"' EXIT
+curl -fSL --retry 3 "${HF_BASE}/manifest.json" -o "${work}/manifest.json" \
+  || die "cannot read ${HF_REPO}/manifest.json"
+
+python3 - "$work/manifest.json" "$NAME" "$SNAP_REF" <<'PY'
+import json, sys
+m = json.load(open(sys.argv[1]))
+if sys.argv[2] in m["models"]:
+    sys.exit(f"error: {sys.argv[2]} already released — released names are immutable")
+if sys.argv[3] and sys.argv[3] not in m["snapshots"]:
+    sys.exit(f"error: --snapshot {sys.argv[3]} is not in the manifest (use --snapshot-dir to register it)")
+PY
+
+SNAP_NAME="$SNAP_REF"
+if [[ -n "$SNAP_DIR" ]]; then
+  SNAP_NAME="$(basename "$SNAP_DIR")"
+  [[ -d "$SNAP_DIR" ]] || die "snapshot dir not found: $SNAP_DIR"
+  python3 - "$work/manifest.json" "$SNAP_NAME" <<'PY'
+import json, sys
+m = json.load(open(sys.argv[1]))
+if sys.argv[2] in m["snapshots"]:
+    sys.exit(f"error: snapshot {sys.argv[2]} already released — snapshots are immutable")
+PY
+fi
+
+# --- upload content first --------------------------------------------------
+
+if [[ -n "$SNAP_DIR" ]]; then
+  step "uploading snapshot ${SNAP_NAME}"
+  hf upload "$HF_REPO" "$SNAP_DIR" "snapshots/${SNAP_NAME}"
+fi
+
+step "uploading models/${NAME}"
+hf upload "$HF_REPO" "$ART_DIR" "models/${NAME}"
+
+if [[ "$APP" == 1 ]]; then
+  step "uploading installer + vendored app"
+  hf upload "$HF_REPO" "${SCRIPT_DIR}/install.sh" install.sh
+  hf upload "$HF_REPO" "${SCRIPT_DIR}/requirements.txt" requirements.txt
+  hf upload "$HF_REPO" "${SCRIPT_DIR}/app" app
+fi
+
+# --- then move the pointer -------------------------------------------------
+
+step "updating manifest"
+python3 - "$work/manifest.json" "$NAME" "$ART_DIR" "$SNAP_NAME" "$SNAP_DIR" \
+          "$DEFAULT" "$EVAL" "${SCRIPT_DIR}/app/PROVENANCE.json" <<'PY'
+import hashlib, json, sys, pathlib, datetime
+mf, name, art, snap_name, snap_dir, default, ev, prov = sys.argv[1:]
+m = json.load(open(mf))
+art = pathlib.Path(art)
+files = {f.name: hashlib.sha256(f.read_bytes()).hexdigest()
+         for f in sorted(art.iterdir()) if f.is_file()}
+service_ver = json.load(open(prov)).get("version", "0.0.0")
+if snap_dir:
+    snap_dir = pathlib.Path(snap_dir)
+    m["snapshots"][snap_name] = {
+        "path": f"snapshots/{snap_name}",
+        "sha256": {str(f.relative_to(snap_dir)): hashlib.sha256(f.read_bytes()).hexdigest()
+                   for f in sorted(snap_dir.rglob("*")) if f.is_file()
+                   and "__pycache__" not in f.parts},
+    }
+if not snap_name:
+    snap_name = m["models"][m["default"]]["snapshot"]
+m["models"][name] = {
+    "path": f"models/{name}",
+    "snapshot": snap_name,
+    "files": files,
+    "min_service": service_ver,
+    "released": datetime.date.today().isoformat(),
+    "eval": {"note": ev} if ev else {},
+}
+if default == "1":
+    m["default"] = name
+json.dump(m, open(mf, "w"), indent=2)
+print(f"  models[{name}] snapshot={snap_name} default={m['default']}")
+PY
+
+hf upload "$HF_REPO" "${work}/manifest.json" manifest.json \
+  --commit-message "release ${NAME}$([[ $DEFAULT == 1 ]] && echo ' (default)')"
+
+step "released"
+echo "name        ${NAME}$([[ $DEFAULT == 1 ]] && echo '  <- new channel default')"
+echo "repo        https://huggingface.co/${HF_REPO}"
+echo "users run   bash install.sh          # picks up the new default"
+echo "            bash install.sh --model ${NAME}"