Spaces:
Running
Running
Download image/build.sh from ereniko/LocalWebAgent: direct link, hf CLI and curl.
- Browser
- Download file 8.3 kB
-
https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/image/build.sh
- Command line
-
hf download hf://spaces/ereniko/LocalWebAgent/image/build.sh
-
curl -L -o build.sh https://huggingface.co/spaces/ereniko/LocalWebAgent/resolve/main/image/build.sh
8.3 kB
| # Builds LocalWebAgent's Linux disk: a Debian 12 (bookworm) i386 ext2 image for CheerpX, | |
| # split into the 4 MiB gzipped chunks + manifest.json the app downloads (see chunk.py). | |
| # | |
| # sudo image/build.sh [workdir] full build into workdir (default /tmp/lwa-image) | |
| # sudo image/build.sh [workdir] --from customize rerun from a stage (bootstrap, install, | |
| # customize, slim, pack, chunk) | |
| # | |
| # Needs: root, debootstrap, e2fsprogs, python3, and an x86 host that can run i386 binaries. | |
| # Behind an HTTPS proxy, set https_proxy and optionally BUILD_CA=/path/to/proxy-ca.pem; | |
| # both are only used during the build and removed from the image. | |
| set -euo pipefail | |
| HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" | |
| REPO="$(dirname "$HERE")" | |
| WORK="${1:-/tmp/lwa-image}" | |
| FROM="${3:-bootstrap}" | |
| [ "${2:-}" = "--from" ] || FROM=bootstrap | |
| R="$WORK/rootfs" | |
| MIRROR="${MIRROR:-https://deb.debian.org/debian}" | |
| # CheerpX's HttpBytesDevice refuses disks over 2 GiB (cx_esm.js checks Content-Range against | |
| # 2147483648), so the whole filesystem, software plus free space, must fit in that. | |
| MAX_MB=2048 | |
| MIN_FREE_MB="${MIN_FREE_MB:-450}" # fail the build if fewer MB would be left for the user's files | |
| STAGES=(bootstrap install customize slim pack chunk) | |
| log() { printf '\033[1;32m==> %s\033[0m\n' "$*"; } | |
| run_in() { chroot "$R" /usr/bin/env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ | |
| HOME=/root LANG=C.UTF-8 DEBIAN_FRONTEND=noninteractive "$@"; } | |
| mounts() { | |
| for m in proc sys dev dev/pts; do mountpoint -q "$R/$m" || mount --bind "/$m" "$R/$m"; done | |
| } | |
| umounts() { | |
| for m in dev/pts dev sys proc; do mountpoint -q "$R/$m" && umount -l "$R/$m" || true; done | |
| } | |
| trap umounts EXIT | |
| should() { # should <stage>: true if <stage> runs given --from | |
| local s seen=0 | |
| for s in "${STAGES[@]}"; do [ "$s" = "$FROM" ] && seen=1; [ "$s" = "$1" ] && return $((1 - seen)); done | |
| return 1 | |
| } | |
| mkdir -p "$WORK" | |
| if should bootstrap; then | |
| log "debootstrap Debian 12 i386" | |
| rm -rf "$R" | |
| debootstrap --arch=i386 --variant=minbase bookworm "$R" "$MIRROR" | |
| fi | |
| if should install; then | |
| log "apt configuration (build-only proxy/CA settings are removed in 'slim')" | |
| cat > "$R/etc/apt/sources.list" <<EOF | |
| deb $MIRROR bookworm main | |
| deb $MIRROR bookworm-updates main | |
| deb ${MIRROR%/debian}/debian-security bookworm-security main | |
| EOF | |
| { | |
| echo 'APT::Install-Recommends "false";' | |
| echo 'APT::Install-Suggests "false";' | |
| if [ -n "${https_proxy:-}" ]; then | |
| echo "Acquire::http::Proxy \"$https_proxy\";" | |
| echo "Acquire::https::Proxy \"$https_proxy\";" | |
| fi | |
| if [ -n "${BUILD_CA:-}" ]; then | |
| cp "$BUILD_CA" "$R/etc/apt/lwa-build-ca.crt" | |
| echo 'Acquire::https::CAInfo "/etc/apt/lwa-build-ca.crt";' | |
| fi | |
| } > "$R/etc/apt/apt.conf.d/99lwa-build" | |
| cp /etc/resolv.conf "$R/etc/resolv.conf" | |
| mounts | |
| log "installing packages ($(grep -v '^#' "$HERE/packages.txt" | wc -w))" | |
| run_in apt-get update -q | |
| run_in apt-get -y -q dist-upgrade | |
| # shellcheck disable=SC2046 | |
| run_in apt-get install -y -q $(grep -v '^#' "$HERE/packages.txt") | |
| umounts | |
| fi | |
| if should customize; then | |
| log "customizing" | |
| mounts | |
| # The VM runs as uid 1000 "user" (see js/vm.js). | |
| run_in id user >/dev/null 2>&1 || run_in useradd -m -u 1000 -s /bin/bash user | |
| cp -a "$HERE/overlay/." "$R/" | |
| run_in chown -R 1000:1000 /home/user | |
| echo localwebagent > "$R/etc/hostname" | |
| # Friendlier names for Debian's renamed tools. | |
| ln -sf /usr/bin/python3 "$R/usr/local/bin/python" | |
| ln -sf /usr/bin/fdfind "$R/usr/local/bin/fd" | |
| ln -sf /usr/bin/batcat "$R/usr/local/bin/bat" | |
| # Environment for the VM's shell (interactive, non-login: it reads /etc/bash.bashrc, | |
| # not /etc/profile.d). Offline Go: never try to download modules. | |
| sed -i '/^# --- LocalWebAgent ---$/,$d' "$R/etc/bash.bashrc" | |
| cat >> "$R/etc/bash.bashrc" <<'EOF' | |
| # --- LocalWebAgent --- | |
| export MPLBACKEND=Agg | |
| # Single-threaded math libraries: thread pools can deadlock on futexes under CheerpX. | |
| export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 NUMEXPR_MAX_THREADS=1 | |
| alias ll='ls -alF --color=auto' la='ls -A --color=auto' l='ls -CF --color=auto' | |
| EOF | |
| # numpy/scipy: use the reference BLAS/LAPACK, not OpenBLAS. OpenBLAS starts a thread pool | |
| # that deadlocks at exit under CheerpX and prints an L2-cache warning on every import. | |
| # (Its packages stay installed: scipy -> pythran depends on them.) | |
| run_in update-alternatives --set libblas.so.3-i386-linux-gnu /usr/lib/i386-linux-gnu/blas/libblas.so.3 | |
| run_in update-alternatives --set liblapack.so.3-i386-linux-gnu /usr/lib/i386-linux-gnu/lapack/liblapack.so.3 | |
| printf 'OPENBLAS_NUM_THREADS=1\nOMP_NUM_THREADS=1\n' > "$R/etc/environment" | |
| # Threading fix for CheerpX (see compat/lwa-compat.c), preloaded into every process. | |
| cp "$HERE/compat/lwa-compat.c" "$R/tmp/lwa-compat.c" | |
| run_in gcc -O2 -Wall -shared -fPIC -o /usr/local/lib/liblwa-compat.so /tmp/lwa-compat.c -pthread | |
| echo /usr/local/lib/liblwa-compat.so > "$R/etc/ld.so.preload" | |
| cat > "$R/tmp/join.c" <<'EOF' | |
| #include <pthread.h> | |
| #include <stdio.h> | |
| #include <unistd.h> | |
| static void *slow(void *a) { usleep(50000); return a; } | |
| int main(void) { pthread_t t; pthread_create(&t, 0, slow, (void *)42); void *r; int e = pthread_join(t, &r); printf("%d %ld\n", e, (long)r); return e; } | |
| EOF | |
| run_in gcc -pthread -o /tmp/join /tmp/join.c | |
| [ "$(run_in /tmp/join)" = "0 42" ] || { echo "lwa-compat self-test failed" >&2; exit 1; } | |
| rm -f "$R"/tmp/join "$R"/tmp/join.c "$R"/tmp/lwa-compat.c | |
| # Precompile Python bytecode so the first import doesn't write to the user's overlay. | |
| run_in python3 -m compileall -q -j 0 /usr/lib/python3/dist-packages /usr/lib/python3.11 >/dev/null || true | |
| # Font cache for matplotlib / graphviz, built once here. | |
| run_in fc-cache -f >/dev/null 2>&1 || true | |
| # Matplotlib's font cache, built where the VM user's matplotlib looks for it. | |
| chroot --userspec=1000:1000 "$R" /usr/bin/env -i HOME=/home/user PATH=/usr/bin:/bin MPLBACKEND=Agg \ | |
| python3 -c "import matplotlib.pyplot" || true | |
| umounts | |
| fi | |
| if should slim; then | |
| log "slimming" | |
| rm -f "$R/etc/apt/apt.conf.d/99lwa-build" "$R/etc/apt/lwa-build-ca.crt" "$R/etc/resolv.conf" | |
| : > "$R/etc/machine-id" | |
| run_in apt-get clean | |
| rm -rf "$R"/var/lib/apt/lists/* "$R"/var/cache/apt/*.bin "$R"/var/cache/debconf/*-old "$R"/var/log/* "$R"/tmp/* "$R"/root/.cache | |
| # Only needed to compile with pythran, which needs network installs anyway. | |
| rm -rf "$R/usr/include/boost" | |
| # Docs: keep the copyright/license files, drop the rest. | |
| find "$R/usr/share/doc" -mindepth 2 ! -name 'copyright' ! -type d -delete 2>/dev/null || true | |
| find "$R/usr/share/doc" -mindepth 1 -type d -empty -delete 2>/dev/null || true | |
| rm -rf "$R/usr/share/info" "$R/usr/share/lintian" "$R/usr/share/linda" | |
| # Translations and non-English man pages. | |
| find "$R/usr/share/locale" -mindepth 1 -maxdepth 1 ! -name 'en*' ! -name 'locale.alias' -exec rm -rf {} + 2>/dev/null || true | |
| find "$R/usr/share/man" -mindepth 1 -maxdepth 1 ! -name 'man*' -exec rm -rf {} + 2>/dev/null || true | |
| # Test suites shipped inside Python packages. Only directories named "tests": numpy.testing, | |
| # sympy.testing and networkx.testing are real modules those libraries import. | |
| for p in numpy scipy pandas matplotlib sympy networkx PIL lxml openpyxl; do | |
| find "$R/usr/lib/python3/dist-packages/$p" -type d -name tests -prune -exec rm -rf {} + 2>/dev/null || true | |
| done | |
| fi | |
| if should pack; then | |
| USED_MB=$(du -sxm "$R" | cut -f1) | |
| SIZE_MB=$MAX_MB | |
| FREE_EST=$(( SIZE_MB - USED_MB * 110 / 100 )) # ext2 metadata takes roughly 10% | |
| log "packing ext2: ${USED_MB} MB used, ${SIZE_MB} MB image, about ${FREE_EST} MB free" | |
| if [ "$FREE_EST" -lt "$MIN_FREE_MB" ]; then | |
| echo "error: only ~${FREE_EST} MB would be free for the user's files (minimum ${MIN_FREE_MB}); remove packages" >&2 | |
| exit 1 | |
| fi | |
| rm -f "$WORK/disk.ext2" | |
| # Same layout as the CheerpX images: ext2, 4 KiB blocks, 256-byte inodes, no reserved blocks. | |
| mke2fs -q -t ext2 -b 4096 -I 256 -m 0 -L lwa-debian12 -d "$R" "$WORK/disk.ext2" "${SIZE_MB}M" | |
| e2fsck -fn "$WORK/disk.ext2" >/dev/null | |
| fi | |
| if should chunk; then | |
| log "chunking into $REPO/disk" | |
| python3 "$HERE/chunk.py" "$WORK/disk.ext2" "$REPO/disk" | |
| fi | |
| log "done" | |