#!/usr/bin/env bash # Publish a model artifact to the Enrell/lain-parser release channel. # # bash release.sh [--default] [--eval "note"] # bash release.sh artifacts/e5-s2 e5-s2 --default \ # --eval "real-final-v1 int8 CPU: 137/196" # # Ordering is the no-break contract: model files land on the Hub first, the # manifest pointer moves last, so a manifest always references already-verified # content. Released names are immutable — a fix ships as a new name. # # --snapshot pair the model with a snapshot already in the manifest # (default: the current default model's snapshot) # --snapshot-dir register + upload a NEW encoder snapshot (only needed # when the model pairs with a snapshot not yet released) # --app also push install.sh, requirements.txt and app/ — # use whenever the vendored inference code changed # # Requires: hf CLI (authenticated, e.g. `hf auth login`), curl, python3. set -euo pipefail HF_REPO="${LAIN_PARSER_HF_REPO:-Enrell/lain-parser}" HF_BASE="https://huggingface.co/${HF_REPO}/resolve/main" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" step() { printf '\n== %s\n' "$*"; } die() { echo "error: $*" >&2; exit 1; } usage() { sed -n '3,18p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 2; } ART_DIR="" NAME="" DEFAULT=0 EVAL="" SNAP_DIR="" SNAP_REF="" APP=0 while [[ $# -gt 0 ]]; do case "$1" in --default) DEFAULT=1; shift ;; --eval) EVAL="$2"; shift 2 ;; --snapshot) SNAP_REF="$2"; shift 2 ;; --snapshot-dir) SNAP_DIR="$2"; shift 2 ;; --app) APP=1; shift ;; -h|--help) usage ;; -*) echo "unknown argument: $1" >&2; exit 2 ;; *) [[ -z "$ART_DIR" ]] && ART_DIR="$1" || NAME="$1"; shift ;; esac done [[ -n "$ART_DIR" && -n "$NAME" ]] || usage [[ "$NAME" =~ ^[a-z0-9][a-z0-9.-]*$ ]] || { echo "bad model name: $NAME" >&2; exit 2; } [[ -f "${ART_DIR}/model.int8.pt" && -f "${ART_DIR}/config.json" ]] || { echo "error: ${ART_DIR} needs model.int8.pt + config.json" >&2; exit 1; } command -v hf >/dev/null || die "hf CLI required (pip install huggingface_hub[cli], then hf auth login)" step "fetching current manifest" work="$(mktemp -d)"; trap 'rm -rf "$work"' EXIT curl -fSL --retry 3 "${HF_BASE}/manifest.json" -o "${work}/manifest.json" \ || die "cannot read ${HF_REPO}/manifest.json" python3 - "$work/manifest.json" "$NAME" "$SNAP_REF" <<'PY' import json, sys m = json.load(open(sys.argv[1])) if sys.argv[2] in m["models"]: sys.exit(f"error: {sys.argv[2]} already released — released names are immutable") if sys.argv[3] and sys.argv[3] not in m["snapshots"]: sys.exit(f"error: --snapshot {sys.argv[3]} is not in the manifest (use --snapshot-dir to register it)") PY SNAP_NAME="$SNAP_REF" if [[ -n "$SNAP_DIR" ]]; then SNAP_NAME="$(basename "$SNAP_DIR")" [[ -d "$SNAP_DIR" ]] || die "snapshot dir not found: $SNAP_DIR" python3 - "$work/manifest.json" "$SNAP_NAME" <<'PY' import json, sys m = json.load(open(sys.argv[1])) if sys.argv[2] in m["snapshots"]: sys.exit(f"error: snapshot {sys.argv[2]} already released — snapshots are immutable") PY fi # --- upload content first -------------------------------------------------- if [[ -n "$SNAP_DIR" ]]; then step "uploading snapshot ${SNAP_NAME}" hf upload "$HF_REPO" "$SNAP_DIR" "snapshots/${SNAP_NAME}" fi step "uploading models/${NAME}" hf upload "$HF_REPO" "$ART_DIR" "models/${NAME}" if [[ "$APP" == 1 ]]; then step "uploading installer + vendored app" hf upload "$HF_REPO" "${SCRIPT_DIR}/install.sh" install.sh hf upload "$HF_REPO" "${SCRIPT_DIR}/requirements.txt" requirements.txt hf upload "$HF_REPO" "${SCRIPT_DIR}/app" app fi # --- then move the pointer ------------------------------------------------- step "updating manifest" python3 - "$work/manifest.json" "$NAME" "$ART_DIR" "$SNAP_NAME" "$SNAP_DIR" \ "$DEFAULT" "$EVAL" "${SCRIPT_DIR}/app/PROVENANCE.json" <<'PY' import hashlib, json, sys, pathlib, datetime mf, name, art, snap_name, snap_dir, default, ev, prov = sys.argv[1:] m = json.load(open(mf)) art = pathlib.Path(art) files = {f.name: hashlib.sha256(f.read_bytes()).hexdigest() for f in sorted(art.iterdir()) if f.is_file()} service_ver = json.load(open(prov)).get("version", "0.0.0") if snap_dir: snap_dir = pathlib.Path(snap_dir) m["snapshots"][snap_name] = { "path": f"snapshots/{snap_name}", "sha256": {str(f.relative_to(snap_dir)): hashlib.sha256(f.read_bytes()).hexdigest() for f in sorted(snap_dir.rglob("*")) if f.is_file() and "__pycache__" not in f.parts}, } if not snap_name: snap_name = m["models"][m["default"]]["snapshot"] m["models"][name] = { "path": f"models/{name}", "snapshot": snap_name, "files": files, "min_service": service_ver, "released": datetime.date.today().isoformat(), "eval": {"note": ev} if ev else {}, } if default == "1": m["default"] = name json.dump(m, open(mf, "w"), indent=2) print(f" models[{name}] snapshot={snap_name} default={m['default']}") PY hf upload "$HF_REPO" "${work}/manifest.json" manifest.json \ --commit-message "release ${NAME}$([[ $DEFAULT == 1 ]] && echo ' (default)')" step "released" echo "name ${NAME}$([[ $DEFAULT == 1 ]] && echo ' <- new channel default')" echo "repo https://huggingface.co/${HF_REPO}" echo "users run bash install.sh # picks up the new default" echo " bash install.sh --model ${NAME}"