Upload install.sh with huggingface_hub

install.sh

@@ -1,21 +1,30 @@
 #!/usr/bin/env bash
-# Install the Lain parser service: one frozen artifact, a private venv and a
-# systemd user unit. Everything lands under --prefix (default
-# ~/.local/share/lain-parser) and ~/.config/systemd/user; no sudo anywhere.
+# Install or update the Lain parser service: one pinned model artifact, a
+# private venv and a systemd user unit. Everything lands under --prefix
+# (default ~/.local/share/lain-parser) and ~/.config/systemd/user; no sudo
+# anywhere.
 #
-#   bash install.sh                     # install, enable and start the unit
-#   bash install.sh --no-start          # install files only
-#   bash install.sh --verify            # re-check the installed hashes
+#   bash install.sh                    # install, or update to the channel default
+#   bash install.sh --model e5-s2      # pin a specific model version
+#   bash install.sh --local            # install from this repo's artifacts/ (dev)
+#   bash install.sh --no-start         # install files only
+#   bash install.sh --verify           # re-check installed hashes + a live parse
 #
-# Source of truth for the artifact and the vendored app: this repository.
+# Artifacts come from the Hugging Face repo `Enrell/lain-parser`. manifest.json
+# there is the release channel: `default` names the model, and every file's
+# sha256 is verified against it before the running unit is touched. A failed
+# download or hash check leaves the installed service untouched.
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO="$(cd "${SCRIPT_DIR}/../.." && pwd)"
-ARTIFACT_NAME="e5-s2"
-ARTIFACT_SRC="${REPO}/artifacts/e5-s2"
-ARTIFACT_SHA="d928668e5ed3c8ea4ac69e42f8d11c90f82120e3307dffaf3a02eb801c077e54"
-SNAPSHOT_SRC="${ANIFILEBERT_SNAPSHOT:-${REPO}/artifacts/anifilebert-dapt512-long}"
+HF_REPO="${LAIN_PARSER_HF_REPO:-Enrell/lain-parser}"
+HF_BASE="https://huggingface.co/${HF_REPO}/resolve/main"
+SERVICE_VERSION="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["version"])' \
+                   "${SCRIPT_DIR}/app/PROVENANCE.json" 2>/dev/null || echo 0.0.0)"
+# --local installs from the repository's artifacts/ tree (development).
+LOCAL=0
+MODEL_ARG=""
 SNAPSHOT_FILES=(anifilebert config.json model.safetensors tokenizer_config.json \
                 vocab.json vocab.char.json label_schema.json dapt.json)
 PREFIX="${LAIN_PARSER_PREFIX:-${HOME}/.local/share/lain-parser}"
@@ -29,7 +38,6 @@ HOSTS="${LAIN_PARSER_HOST:-127.0.0.1,172.17.0.1}"
 SOCKET="${LAIN_PARSER_SOCKET:-${HOME}/.local/share/lain/parser.sock}"
 START=1
 VERIFY=0
-FORCE=0
 
 while [[ $# -gt 0 ]]; do
   case "$1" in
@@ -37,54 +45,163 @@ while [[ $# -gt 0 ]]; do
     --port) PORT="$2"; shift 2 ;;
     --host) HOSTS="$2"; shift 2 ;;
     --socket) SOCKET="$2"; shift 2 ;;
+    --model) MODEL_ARG="$2"; shift 2 ;;
+    --local) LOCAL=1; shift ;;
     --no-start) START=0; shift ;;
-    --force) FORCE=1; shift ;;
     --verify) VERIFY=1; shift ;;
-    -h|--help) sed -n '2,10p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
+    -h|--help) sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
     *) echo "unknown argument: $1" >&2; exit 2 ;;
   esac
 done
 
 step() { printf '\n== %s\n' "$*"; }
 die() { echo "error: $*" >&2; exit 1; }
-
 sha_of() { sha256sum "$1" | cut -d' ' -f1; }
 
-verify_installed() {
-  step "verifying ${PREFIX}"
-  local venv="${PREFIX}/venv/bin/python"
-  [[ -x "$venv" ]] || die "venv missing under ${PREFIX}"
-  local sum installed
-  sum="$(sha_of "${PREFIX}/artifact/${ARTIFACT_NAME}/model.int8.pt")"
-  [[ "$sum" == "$ARTIFACT_SHA" ]] || die "artifact hash mismatch: ${sum}"
-  echo "artifact ${ARTIFACT_NAME} hash ok"
-  "$venv" "${PREFIX}/app/serve.py" --check "Example.Show.S01E02.1080p-GROUP.mkv" \
-    --artifact "${PREFIX}/artifact/${ARTIFACT_NAME}" --snapshot "${PREFIX}/anifilebert" \
-    >/dev/null
-  echo "model loads and parses"
+command -v python3 >/dev/null || die "python3 is required"
+command -v curl >/dev/null || [[ "$LOCAL" == 1 ]] || die "curl is required for downloads (or use --local)"
+
+# --- resolve the model + snapshot -----------------------------------------
+
+MANIFEST_LOCAL="${PREFIX}/manifest.installed.json"
+
+hf_fetch() { # hf_fetch <repo-path> <dest> — anonymous resolve-URL download
+  curl -fSL --retry 3 --connect-timeout 10 "${HF_BASE}/$1" -o "$2"
+}
+
+read_manifest() { # read_manifest <file> <python-expression-over-m>
+  python3 - "$1" "$2" <<'PY'
+import json, sys
+m = json.load(open(sys.argv[1]))
+print(eval(sys.argv[2]))  # manifest is our own release file; trusted shape
+PY
 }
 
 if [[ "$VERIFY" == 1 ]]; then
-  verify_installed
+  step "verifying ${PREFIX}"
+  [[ -x "${PREFIX}/venv/bin/python" ]] || die "venv missing under ${PREFIX}"
+  [[ -f "$MANIFEST_LOCAL" ]] || die "installed manifest missing — was this installed by this installer?"
+  installed_name="$(read_manifest "$MANIFEST_LOCAL" 'm["installed_model"]')"
+  want="$(read_manifest "$MANIFEST_LOCAL" "m['models']['${installed_name}']['files']['model.int8.pt']")"
+  sum="$(sha_of "${PREFIX}/artifact/${installed_name}/model.int8.pt")"
+  [[ "$sum" == "$want" ]] || die "artifact hash mismatch: ${sum} != ${want}"
+  echo "artifact ${installed_name} hash ok"
+  "${PREFIX}/venv/bin/python" "${PREFIX}/app/serve.py" --check \
+    "Example.Show.S01E02.1080p-GROUP.mkv" \
+    --artifact "${PREFIX}/artifact/${installed_name}" \
+    --snapshot "${PREFIX}/anifilebert" >/dev/null
+  echo "model loads and parses"
   exit 0
 fi
 
-[[ -f "${ARTIFACT_SRC}/model.int8.pt" ]] || die "artifact not found at ${ARTIFACT_SRC}"
-[[ -f "${ARTIFACT_SRC}/config.json" ]] || die "artifact config not found at ${ARTIFACT_SRC}"
-sum="$(sha_of "${ARTIFACT_SRC}/model.int8.pt")"
-[[ "$sum" == "$ARTIFACT_SHA" ]] || die "source artifact hash mismatch: ${sum}"
-[[ -d "$SNAPSHOT_SRC" ]] || die "snapshot not found at ${SNAPSHOT_SRC}"
+if [[ "$LOCAL" == 1 ]]; then
+  ARTIFACT_NAME="${MODEL_ARG:-e5-s2}"
+  ARTIFACT_SRC="${REPO}/artifacts/${ARTIFACT_NAME}"
+  SNAPSHOT_SRC="${ANIFILEBERT_SNAPSHOT:-${REPO}/artifacts/anifilebert-dapt512-long}"
+  [[ -f "${ARTIFACT_SRC}/model.int8.pt" ]] || die "local artifact not found at ${ARTIFACT_SRC}"
+  [[ -f "${ARTIFACT_SRC}/config.json" ]] || die "local artifact config not found"
+  [[ -d "$SNAPSHOT_SRC" ]] || die "local snapshot not found at ${SNAPSHOT_SRC}"
+  MANIFEST_TMP=""
+else
+  step "fetching release manifest (${HF_REPO})"
+  mkdir -p "${PREFIX}/.download"
+  MANIFEST_TMP="${PREFIX}/.download/manifest.json"
+  hf_fetch "manifest.json" "$MANIFEST_TMP" || die "manifest download failed"
+  ARTIFACT_NAME="${MODEL_ARG:-$(read_manifest "$MANIFEST_TMP" 'm["default"]')}"
+  read_manifest "$MANIFEST_TMP" "'${ARTIFACT_NAME}' in m['models']" >/dev/null 2>&1 || \
+    die "model ${ARTIFACT_NAME} not in manifest"
+  # Refuse cleanly when a model needs a newer service than the vendored app ships.
+  need="$(read_manifest "$MANIFEST_TMP" "m['models']['${ARTIFACT_NAME}'].get('min_service','0')")"
+  python3 - "$need" "$SERVICE_VERSION" <<'PY' || die "model requires service >= $need; update the installer first"
+import sys
+key = lambda v: [int(x) for x in v.split('.')]
+sys.exit(0 if key(sys.argv[2]) >= key(sys.argv[1]) else 1)
+PY
+  SNAP_NAME="$(read_manifest "$MANIFEST_TMP" "m['models']['${ARTIFACT_NAME}']['snapshot']")"
+  echo "channel default: $(read_manifest "$MANIFEST_TMP" 'm["default"]') -> installing ${ARTIFACT_NAME} (snapshot ${SNAP_NAME})"
+fi
 
 step "installing into ${PREFIX}"
-mkdir -p "${PREFIX}/app/fp" "${PREFIX}/artifact/${ARTIFACT_NAME}" "${PREFIX}/anifilebert"
+mkdir -p "${PREFIX}/app/fp" "${PREFIX}/artifact" "${PREFIX}/anifilebert"
 cp -f "${SCRIPT_DIR}/app/serve.py" "${PREFIX}/app/serve.py"
 cp -f "${SCRIPT_DIR}"/app/fp/*.py "${PREFIX}/app/fp/"
 cp -f "${SCRIPT_DIR}/requirements.txt" "${PREFIX}/requirements.txt"
-cp -f "${ARTIFACT_SRC}/model.int8.pt" "${ARTIFACT_SRC}/config.json" \
-      "${PREFIX}/artifact/${ARTIFACT_NAME}/"
-for entry in "${SNAPSHOT_FILES[@]}"; do
-  cp -RL "${SNAPSHOT_SRC}/${entry}" "${PREFIX}/anifilebert/"
-done
+
+if [[ "$LOCAL" == 1 ]]; then
+  cp -f "${ARTIFACT_SRC}/model.int8.pt" "${ARTIFACT_SRC}/config.json" \
+        "${PREFIX}/artifact/${ARTIFACT_NAME}/" 2>/dev/null || {
+    mkdir -p "${PREFIX}/artifact/${ARTIFACT_NAME}"
+    cp -f "${ARTIFACT_SRC}/model.int8.pt" "${ARTIFACT_SRC}/config.json" \
+          "${PREFIX}/artifact/${ARTIFACT_NAME}/"; }
+  for entry in "${SNAPSHOT_FILES[@]}"; do
+    cp -RL "${SNAPSHOT_SRC}/${entry}" "${PREFIX}/anifilebert/" 2>/dev/null || true
+  done
+else
+  # Download each needed file to a staging dir, verify sha256 against the
+  # manifest, then move into place. Nothing below $PREFIX/artifact or
+  # $PREFIX/anifilebert changes until every hash checks out.
+  stage="${PREFIX}/.download/${ARTIFACT_NAME}"
+  mkdir -p "${stage}/model" "${stage}/snapshot"
+  while read -r f sha; do
+    [[ "$f" == "model.int8.pt" || "$f" == "config.json" ]] || continue
+    dest="${stage}/model/${f}"
+    installed="${PREFIX}/artifact/${ARTIFACT_NAME}/${f}"
+    if [[ -f "$dest" && "$(sha_of "$dest")" == "$sha" ]] \
+       || { [[ -f "$installed" && "$(sha_of "$installed")" == "$sha" ]] \
+            && cp -f "$installed" "$dest"; }; then
+      echo "already have ${f}"; continue
+    fi
+    echo "fetching models/${ARTIFACT_NAME}/${f}"
+    hf_fetch "models/${ARTIFACT_NAME}/${f}" "$dest" || die "download failed: $f"
+    [[ "$(sha_of "$dest")" == "$sha" ]] || die "hash mismatch after download: $f"
+  done < <(read_manifest "$MANIFEST_TMP" \
+           "'\n'.join(f'{k} {v}' for k,v in m['models']['${ARTIFACT_NAME}']['files'].items())")
+  # snapshot files the service needs: the anifilebert/ package plus the
+  # top-level files listed in SNAPSHOT_FILES (dapt.json tolerated if absent).
+  needed="$(printf '%s\n' "${SNAPSHOT_FILES[@]}" | grep -v '^anifilebert$')"
+  while read -r f sha; do
+    [[ -n "$f" ]] || continue
+    dest="${stage}/snapshot/${f}"
+    mkdir -p "$(dirname "$dest")"
+    installed="${PREFIX}/anifilebert/${f}"
+    if [[ -f "$dest" && "$(sha_of "$dest")" == "$sha" ]] \
+       || { [[ -f "$installed" && "$(sha_of "$installed")" == "$sha" ]] \
+            && cp -f "$installed" "$dest"; }; then
+      echo "already have snapshot/${f}"; continue
+    fi
+    echo "fetching snapshots/${SNAP_NAME}/${f}"
+    if hf_fetch "snapshots/${SNAP_NAME}/${f}" "$dest" && [[ "$(sha_of "$dest")" == "$sha" ]]; then
+      continue
+    fi
+    if [[ "$f" == "dapt.json" ]]; then
+      echo "warning: ${f} unavailable — skipping (provenance only)"
+      rm -f "$dest"; continue
+    fi
+    die "snapshot file ${f} failed to fetch or verify"
+  done < <(python3 - "$MANIFEST_TMP" "$SNAP_NAME" $needed <<'PY'
+import json, sys
+m = json.load(open(sys.argv[1]))
+snap = m["snapshots"][sys.argv[2]]
+want = set(sys.argv[3:])
+for rel, sha in sorted(snap["sha256"].items()):
+    if rel in want or rel.startswith("anifilebert/"):
+        print(rel, sha)
+PY
+)
+  mkdir -p "${PREFIX}/artifact/${ARTIFACT_NAME}"
+  cp -f "${stage}/model/model.int8.pt" "${stage}/model/config.json" \
+        "${PREFIX}/artifact/${ARTIFACT_NAME}/"
+  cp -Rf "${stage}/snapshot/." "${PREFIX}/anifilebert/"
+fi
+
+# Record the manifest used for this install (offline verify + update checks).
+if [[ -n "${MANIFEST_TMP}" ]]; then
+  python3 - "$MANIFEST_TMP" "$MANIFEST_LOCAL" "$ARTIFACT_NAME" <<'PY'
+import json, sys
+m = json.load(open(sys.argv[1])); m["installed_model"] = sys.argv[3]
+json.dump(m, open(sys.argv[2], "w"), indent=2)
+PY
+fi
 
 step "recording hashes"
 {
@@ -113,6 +230,13 @@ else
 fi
 "${PREFIX}/venv/bin/python" -c "import torch, transformers; print('torch', torch.__version__, '| transformers', transformers.__version__)"
 
+step "smoke test before touching the service"
+"${PREFIX}/venv/bin/python" "${PREFIX}/app/serve.py" --check \
+  "Example.Show.S01E02.1080p-GROUP.mkv" \
+  --artifact "${PREFIX}/artifact/${ARTIFACT_NAME}" \
+  --snapshot "${PREFIX}/anifilebert" >/dev/null \
+  || die "new artifact failed the smoke parse — the running service was left alone"
+
 step "systemd user unit"
 mkdir -p "${UNIT_DIR}"
 unit_tmp="$(mktemp)"
@@ -137,18 +261,18 @@ if [[ ! -f "${UNIT_DIR}/${UNIT}" ]]; then
   echo "wrote ${UNIT_DIR}/${UNIT}"
 elif diff -q "$unit_tmp" "${UNIT_DIR}/${UNIT}" >/dev/null; then
   echo "unit already up to date"
-elif [[ "$FORCE" == 1 ]]; then
-  install -m 0644 "$unit_tmp" "${UNIT_DIR}/${UNIT}"
-  echo "unit replaced (--force)"
 else
-  install -m 0644 "$unit_tmp" "${UNIT_DIR}/${UNIT}.new"
-  echo "unit differs: wrote ${UNIT_DIR}/${UNIT}.new for review (or pass --force)"
+  # A model bump changes ExecStart; keep the old unit as .bak for inspection.
+  cp -f "${UNIT_DIR}/${UNIT}" "${UNIT_DIR}/${UNIT}.bak"
+  install -m 0644 "$unit_tmp" "${UNIT_DIR}/${UNIT}"
+  echo "unit updated (old copy: ${UNIT}.bak)"
 fi
 rm -f "$unit_tmp"
 
 if [[ "$START" == 1 ]] && command -v systemctl >/dev/null 2>&1; then
   systemctl --user daemon-reload
   systemctl --user enable --now "$UNIT" || true
+  systemctl --user restart "$UNIT" || true
   if command -v curl >/dev/null 2>&1; then
     ok=""
     for _ in $(seq 1 20); do
@@ -168,10 +292,13 @@ fi
 step "done"
 cat <<SUMMARY
 prefix      ${PREFIX}
-artifact    ${ARTIFACT_NAME} (${ARTIFACT_SHA:0:16}…)
+artifact    ${ARTIFACT_NAME}
+source      $([[ "$LOCAL" == 1 ]] && echo "local artifacts/" || echo "${HF_REPO} (manifest)")
 unit        ${UNIT_DIR}/${UNIT}
 native URL  http://127.0.0.1:${PORT}
 docker URL  http://172.17.0.1:${PORT}   (or host.docker.internal with host-gateway)
 docker UDS  unix:///data/parser.sock    (socket at ${SOCKET})
+update      bash install.sh            (re-run: installs the channel default)
+rollback    bash install.sh --model <previous-name>
 verify      bash install.sh --verify
 SUMMARY