Upload release.sh with huggingface_hub

dcff182 verified 22 days ago

5.57 kB

#!/usr/bin/env bash
# Publish a model artifact to the Enrell/lain-parser release channel.
#
#   bash release.sh <artifact-dir> <name> [--default] [--eval "note"]
#   bash release.sh artifacts/e5-s2 e5-s2 --default \
#       --eval "real-final-v1 int8 CPU: 137/196"
#
# Ordering is the no-break contract: model files land on the Hub first, the
# manifest pointer moves last, so a manifest always references already-verified
# content. Released names are immutable — a fix ships as a new name.
#
#   --snapshot <name>      pair the model with a snapshot already in the manifest
#                          (default: the current default model's snapshot)
#   --snapshot-dir <dir>   register + upload a NEW encoder snapshot (only needed
#                          when the model pairs with a snapshot not yet released)
#   --app                  also push install.sh, requirements.txt and app/ —
#                          use whenever the vendored inference code changed
#
# Requires: hf CLI (authenticated, e.g. `hf auth login`), curl, python3.
set -euo pipefail

HF_REPO="${LAIN_PARSER_HF_REPO:-Enrell/lain-parser}"
HF_BASE="https://huggingface.co/${HF_REPO}/resolve/main"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

step() { printf '\n== %s\n' "$*"; }
die() { echo "error: $*" >&2; exit 1; }
usage() { sed -n '3,18p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; }

ART_DIR="" NAME="" DEFAULT=0 EVAL="" SNAP_DIR="" SNAP_REF="" APP=0
while [[ $# -gt 0 ]]; do
  case "$1" in
    --default) DEFAULT=1; shift ;;
    --eval) EVAL="$2"; shift 2 ;;
    --snapshot) SNAP_REF="$2"; shift 2 ;;
    --snapshot-dir) SNAP_DIR="$2"; shift 2 ;;
    --app) APP=1; shift ;;
    -h|--help) usage ;;
    -*) echo "unknown argument: $1" >&2; exit 2 ;;
    *) [[ -z "$ART_DIR" ]] && ART_DIR="$1" || NAME="$1"; shift ;;
  esac
done
[[ -n "$ART_DIR" && -n "$NAME" ]] || usage
[[ "$NAME" =~ ^[a-z0-9][a-z0-9.-]*$ ]] || { echo "bad model name: $NAME" >&2; exit 2; }
[[ -f "${ART_DIR}/model.int8.pt" && -f "${ART_DIR}/config.json" ]] || {
  echo "error: ${ART_DIR} needs model.int8.pt + config.json" >&2; exit 1; }

command -v hf >/dev/null || die "hf CLI required (pip install huggingface_hub[cli], then hf auth login)"

step "fetching current manifest"
work="$(mktemp -d)"; trap 'rm -rf "$work"' EXIT
curl -fSL --retry 3 "${HF_BASE}/manifest.json" -o "${work}/manifest.json" \
  || die "cannot read ${HF_REPO}/manifest.json"

python3 - "$work/manifest.json" "$NAME" "$SNAP_REF" <<'PY'
import json, sys
m = json.load(open(sys.argv[1]))
if sys.argv[2] in m["models"]:
    sys.exit(f"error: {sys.argv[2]} already released — released names are immutable")
if sys.argv[3] and sys.argv[3] not in m["snapshots"]:
    sys.exit(f"error: --snapshot {sys.argv[3]} is not in the manifest (use --snapshot-dir to register it)")
PY

SNAP_NAME="$SNAP_REF"
if [[ -n "$SNAP_DIR" ]]; then
  SNAP_NAME="$(basename "$SNAP_DIR")"
  [[ -d "$SNAP_DIR" ]] || die "snapshot dir not found: $SNAP_DIR"
  python3 - "$work/manifest.json" "$SNAP_NAME" <<'PY'
import json, sys
m = json.load(open(sys.argv[1]))
if sys.argv[2] in m["snapshots"]:
    sys.exit(f"error: snapshot {sys.argv[2]} already released — snapshots are immutable")
PY
fi

# --- upload content first --------------------------------------------------

if [[ -n "$SNAP_DIR" ]]; then
  step "uploading snapshot ${SNAP_NAME}"
  hf upload "$HF_REPO" "$SNAP_DIR" "snapshots/${SNAP_NAME}"
fi

step "uploading models/${NAME}"
hf upload "$HF_REPO" "$ART_DIR" "models/${NAME}"

if [[ "$APP" == 1 ]]; then
  step "uploading installer + vendored app"
  hf upload "$HF_REPO" "${SCRIPT_DIR}/install.sh" install.sh
  hf upload "$HF_REPO" "${SCRIPT_DIR}/requirements.txt" requirements.txt
  hf upload "$HF_REPO" "${SCRIPT_DIR}/app" app
fi

# --- then move the pointer -------------------------------------------------

step "updating manifest"
python3 - "$work/manifest.json" "$NAME" "$ART_DIR" "$SNAP_NAME" "$SNAP_DIR" \
          "$DEFAULT" "$EVAL" "${SCRIPT_DIR}/app/PROVENANCE.json" <<'PY'
import hashlib, json, sys, pathlib, datetime
mf, name, art, snap_name, snap_dir, default, ev, prov = sys.argv[1:]
m = json.load(open(mf))
art = pathlib.Path(art)
files = {f.name: hashlib.sha256(f.read_bytes()).hexdigest()
         for f in sorted(art.iterdir()) if f.is_file()}
service_ver = json.load(open(prov)).get("version", "0.0.0")
if snap_dir:
    snap_dir = pathlib.Path(snap_dir)
    m["snapshots"][snap_name] = {
        "path": f"snapshots/{snap_name}",
        "sha256": {str(f.relative_to(snap_dir)): hashlib.sha256(f.read_bytes()).hexdigest()
                   for f in sorted(snap_dir.rglob("*")) if f.is_file()
                   and "__pycache__" not in f.parts},
    }
if not snap_name:
    snap_name = m["models"][m["default"]]["snapshot"]
m["models"][name] = {
    "path": f"models/{name}",
    "snapshot": snap_name,
    "files": files,
    "min_service": service_ver,
    "released": datetime.date.today().isoformat(),
    "eval": {"note": ev} if ev else {},
}
if default == "1":
    m["default"] = name
json.dump(m, open(mf, "w"), indent=2)
print(f"  models[{name}] snapshot={snap_name} default={m['default']}")
PY

hf upload "$HF_REPO" "${work}/manifest.json" manifest.json \
  --commit-message "release ${NAME}$([[ $DEFAULT == 1 ]] && echo ' (default)')"

step "released"
echo "name        ${NAME}$([[ $DEFAULT == 1 ]] && echo '  <- new channel default')"
echo "repo        https://huggingface.co/${HF_REPO}"
echo "users run   bash install.sh          # picks up the new default"
echo "            bash install.sh --model ${NAME}"

	#!/usr/bin/env bash
	# Publish a model artifact to the Enrell/lain-parser release channel.
	#
	# bash release.sh <artifact-dir> <name> [--default] [--eval "note"]
	# bash release.sh artifacts/e5-s2 e5-s2 --default \
	# --eval "real-final-v1 int8 CPU: 137/196"
	#
	# Ordering is the no-break contract: model files land on the Hub first, the
	# manifest pointer moves last, so a manifest always references already-verified
	# content. Released names are immutable — a fix ships as a new name.
	#
	# --snapshot <name> pair the model with a snapshot already in the manifest
	# (default: the current default model's snapshot)
	# --snapshot-dir <dir> register + upload a NEW encoder snapshot (only needed
	# when the model pairs with a snapshot not yet released)
	# --app also push install.sh, requirements.txt and app/ —
	# use whenever the vendored inference code changed
	#
	# Requires: hf CLI (authenticated, e.g. `hf auth login`), curl, python3.
	set -euo pipefail

	HF_REPO="${LAIN_PARSER_HF_REPO:-Enrell/lain-parser}"
	HF_BASE="https://huggingface.co/${HF_REPO}/resolve/main"
	SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

	step() { printf '\n== %s\n' "$*"; }
	die() { echo "error: $*" >&2; exit 1; }
	usage() { sed -n '3,18p' "$0" \| sed 's/^# \{0,1\}//' >&2; exit 2; }

	ART_DIR="" NAME="" DEFAULT=0 EVAL="" SNAP_DIR="" SNAP_REF="" APP=0
	while [[ $# -gt 0 ]]; do
	case "$1" in
	--default) DEFAULT=1; shift ;;
	--eval) EVAL="$2"; shift 2 ;;
	--snapshot) SNAP_REF="$2"; shift 2 ;;
	--snapshot-dir) SNAP_DIR="$2"; shift 2 ;;
	--app) APP=1; shift ;;
	-h\|--help) usage ;;
	-*) echo "unknown argument: $1" >&2; exit 2 ;;
	*) [[ -z "$ART_DIR" ]] && ART_DIR="$1" \|\| NAME="$1"; shift ;;
	esac
	done
	[[ -n "$ART_DIR" && -n "$NAME" ]] \|\| usage
	[[ "$NAME" =~ ^[a-z0-9][a-z0-9.-]*$ ]] \|\| { echo "bad model name: $NAME" >&2; exit 2; }
	[[ -f "${ART_DIR}/model.int8.pt" && -f "${ART_DIR}/config.json" ]] \|\| {
	echo "error: ${ART_DIR} needs model.int8.pt + config.json" >&2; exit 1; }

	command -v hf >/dev/null \|\| die "hf CLI required (pip install huggingface_hub[cli], then hf auth login)"

	step "fetching current manifest"
	work="$(mktemp -d)"; trap 'rm -rf "$work"' EXIT
	curl -fSL --retry 3 "${HF_BASE}/manifest.json" -o "${work}/manifest.json" \
	\|\| die "cannot read ${HF_REPO}/manifest.json"

	python3 - "$work/manifest.json" "$NAME" "$SNAP_REF" <<'PY'
	import json, sys
	m = json.load(open(sys.argv[1]))
	if sys.argv[2] in m["models"]:
	sys.exit(f"error: {sys.argv[2]} already released — released names are immutable")
	if sys.argv[3] and sys.argv[3] not in m["snapshots"]:
	sys.exit(f"error: --snapshot {sys.argv[3]} is not in the manifest (use --snapshot-dir to register it)")
	PY

	SNAP_NAME="$SNAP_REF"
	if [[ -n "$SNAP_DIR" ]]; then
	SNAP_NAME="$(basename "$SNAP_DIR")"
	[[ -d "$SNAP_DIR" ]] \|\| die "snapshot dir not found: $SNAP_DIR"
	python3 - "$work/manifest.json" "$SNAP_NAME" <<'PY'
	import json, sys
	m = json.load(open(sys.argv[1]))
	if sys.argv[2] in m["snapshots"]:
	sys.exit(f"error: snapshot {sys.argv[2]} already released — snapshots are immutable")
	PY
	fi

	# --- upload content first --------------------------------------------------

	if [[ -n "$SNAP_DIR" ]]; then
	step "uploading snapshot ${SNAP_NAME}"
	hf upload "$HF_REPO" "$SNAP_DIR" "snapshots/${SNAP_NAME}"
	fi

	step "uploading models/${NAME}"
	hf upload "$HF_REPO" "$ART_DIR" "models/${NAME}"

	if [[ "$APP" == 1 ]]; then
	step "uploading installer + vendored app"
	hf upload "$HF_REPO" "${SCRIPT_DIR}/install.sh" install.sh
	hf upload "$HF_REPO" "${SCRIPT_DIR}/requirements.txt" requirements.txt
	hf upload "$HF_REPO" "${SCRIPT_DIR}/app" app
	fi

	# --- then move the pointer -------------------------------------------------

	step "updating manifest"
	python3 - "$work/manifest.json" "$NAME" "$ART_DIR" "$SNAP_NAME" "$SNAP_DIR" \
	"$DEFAULT" "$EVAL" "${SCRIPT_DIR}/app/PROVENANCE.json" <<'PY'
	import hashlib, json, sys, pathlib, datetime
	mf, name, art, snap_name, snap_dir, default, ev, prov = sys.argv[1:]
	m = json.load(open(mf))
	art = pathlib.Path(art)
	files = {f.name: hashlib.sha256(f.read_bytes()).hexdigest()
	for f in sorted(art.iterdir()) if f.is_file()}
	service_ver = json.load(open(prov)).get("version", "0.0.0")
	if snap_dir:
	snap_dir = pathlib.Path(snap_dir)
	m["snapshots"][snap_name] = {
	"path": f"snapshots/{snap_name}",
	"sha256": {str(f.relative_to(snap_dir)): hashlib.sha256(f.read_bytes()).hexdigest()
	for f in sorted(snap_dir.rglob("*")) if f.is_file()
	and "__pycache__" not in f.parts},
	}
	if not snap_name:
	snap_name = m["models"][m["default"]]["snapshot"]
	m["models"][name] = {
	"path": f"models/{name}",
	"snapshot": snap_name,
	"files": files,
	"min_service": service_ver,
	"released": datetime.date.today().isoformat(),
	"eval": {"note": ev} if ev else {},
	}
	if default == "1":
	m["default"] = name
	json.dump(m, open(mf, "w"), indent=2)
	print(f" models[{name}] snapshot={snap_name} default={m['default']}")
	PY

	hf upload "$HF_REPO" "${work}/manifest.json" manifest.json \
	--commit-message "release ${NAME}$([[ $DEFAULT == 1 ]] && echo ' (default)')"

	step "released"
	echo "name ${NAME}$([[ $DEFAULT == 1 ]] && echo ' <- new channel default')"
	echo "repo https://huggingface.co/${HF_REPO}"
	echo "users run bash install.sh # picks up the new default"
	echo " bash install.sh --model ${NAME}"