#!/bin/bash # Builds LocalWebAgent's Linux disk: a Debian 12 (bookworm) i386 ext2 image for CheerpX, # split into the 4 MiB gzipped chunks + manifest.json the app downloads (see chunk.py). # # sudo image/build.sh [workdir] full build into workdir (default /tmp/lwa-image) # sudo image/build.sh [workdir] --from customize rerun from a stage (bootstrap, install, # customize, slim, pack, chunk) # # Needs: root, debootstrap, e2fsprogs, python3, and an x86 host that can run i386 binaries. # Behind an HTTPS proxy, set https_proxy and optionally BUILD_CA=/path/to/proxy-ca.pem; # both are only used during the build and removed from the image. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO="$(dirname "$HERE")" WORK="${1:-/tmp/lwa-image}" FROM="${3:-bootstrap}" [ "${2:-}" = "--from" ] || FROM=bootstrap R="$WORK/rootfs" MIRROR="${MIRROR:-https://deb.debian.org/debian}" # CheerpX's HttpBytesDevice refuses disks over 2 GiB (cx_esm.js checks Content-Range against # 2147483648), so the whole filesystem, software plus free space, must fit in that. MAX_MB=2048 MIN_FREE_MB="${MIN_FREE_MB:-450}" # fail the build if fewer MB would be left for the user's files STAGES=(bootstrap install customize slim pack chunk) log() { printf '\033[1;32m==> %s\033[0m\n' "$*"; } run_in() { chroot "$R" /usr/bin/env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ HOME=/root LANG=C.UTF-8 DEBIAN_FRONTEND=noninteractive "$@"; } mounts() { for m in proc sys dev dev/pts; do mountpoint -q "$R/$m" || mount --bind "/$m" "$R/$m"; done } umounts() { for m in dev/pts dev sys proc; do mountpoint -q "$R/$m" && umount -l "$R/$m" || true; done } trap umounts EXIT should() { # should : true if runs given --from local s seen=0 for s in "${STAGES[@]}"; do [ "$s" = "$FROM" ] && seen=1; [ "$s" = "$1" ] && return $((1 - seen)); done return 1 } mkdir -p "$WORK" if should bootstrap; then log "debootstrap Debian 12 i386" rm -rf "$R" debootstrap --arch=i386 --variant=minbase bookworm "$R" "$MIRROR" fi if should install; then log "apt configuration (build-only proxy/CA settings are removed in 'slim')" cat > "$R/etc/apt/sources.list" < "$R/etc/apt/apt.conf.d/99lwa-build" cp /etc/resolv.conf "$R/etc/resolv.conf" mounts log "installing packages ($(grep -v '^#' "$HERE/packages.txt" | wc -w))" run_in apt-get update -q run_in apt-get -y -q dist-upgrade # shellcheck disable=SC2046 run_in apt-get install -y -q $(grep -v '^#' "$HERE/packages.txt") umounts fi if should customize; then log "customizing" mounts # The VM runs as uid 1000 "user" (see js/vm.js). run_in id user >/dev/null 2>&1 || run_in useradd -m -u 1000 -s /bin/bash user cp -a "$HERE/overlay/." "$R/" run_in chown -R 1000:1000 /home/user echo localwebagent > "$R/etc/hostname" # Friendlier names for Debian's renamed tools. ln -sf /usr/bin/python3 "$R/usr/local/bin/python" ln -sf /usr/bin/fdfind "$R/usr/local/bin/fd" ln -sf /usr/bin/batcat "$R/usr/local/bin/bat" # Environment for the VM's shell (interactive, non-login: it reads /etc/bash.bashrc, # not /etc/profile.d). Offline Go: never try to download modules. sed -i '/^# --- LocalWebAgent ---$/,$d' "$R/etc/bash.bashrc" cat >> "$R/etc/bash.bashrc" <<'EOF' # --- LocalWebAgent --- export MPLBACKEND=Agg # Single-threaded math libraries: thread pools can deadlock on futexes under CheerpX. export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 NUMEXPR_MAX_THREADS=1 alias ll='ls -alF --color=auto' la='ls -A --color=auto' l='ls -CF --color=auto' EOF # numpy/scipy: use the reference BLAS/LAPACK, not OpenBLAS. OpenBLAS starts a thread pool # that deadlocks at exit under CheerpX and prints an L2-cache warning on every import. # (Its packages stay installed: scipy -> pythran depends on them.) run_in update-alternatives --set libblas.so.3-i386-linux-gnu /usr/lib/i386-linux-gnu/blas/libblas.so.3 run_in update-alternatives --set liblapack.so.3-i386-linux-gnu /usr/lib/i386-linux-gnu/lapack/liblapack.so.3 printf 'OPENBLAS_NUM_THREADS=1\nOMP_NUM_THREADS=1\n' > "$R/etc/environment" # Threading fix for CheerpX (see compat/lwa-compat.c), preloaded into every process. cp "$HERE/compat/lwa-compat.c" "$R/tmp/lwa-compat.c" run_in gcc -O2 -Wall -shared -fPIC -o /usr/local/lib/liblwa-compat.so /tmp/lwa-compat.c -pthread echo /usr/local/lib/liblwa-compat.so > "$R/etc/ld.so.preload" cat > "$R/tmp/join.c" <<'EOF' #include #include #include static void *slow(void *a) { usleep(50000); return a; } int main(void) { pthread_t t; pthread_create(&t, 0, slow, (void *)42); void *r; int e = pthread_join(t, &r); printf("%d %ld\n", e, (long)r); return e; } EOF run_in gcc -pthread -o /tmp/join /tmp/join.c [ "$(run_in /tmp/join)" = "0 42" ] || { echo "lwa-compat self-test failed" >&2; exit 1; } rm -f "$R"/tmp/join "$R"/tmp/join.c "$R"/tmp/lwa-compat.c # Precompile Python bytecode so the first import doesn't write to the user's overlay. run_in python3 -m compileall -q -j 0 /usr/lib/python3/dist-packages /usr/lib/python3.11 >/dev/null || true # Font cache for matplotlib / graphviz, built once here. run_in fc-cache -f >/dev/null 2>&1 || true # Matplotlib's font cache, built where the VM user's matplotlib looks for it. chroot --userspec=1000:1000 "$R" /usr/bin/env -i HOME=/home/user PATH=/usr/bin:/bin MPLBACKEND=Agg \ python3 -c "import matplotlib.pyplot" || true umounts fi if should slim; then log "slimming" rm -f "$R/etc/apt/apt.conf.d/99lwa-build" "$R/etc/apt/lwa-build-ca.crt" "$R/etc/resolv.conf" : > "$R/etc/machine-id" run_in apt-get clean rm -rf "$R"/var/lib/apt/lists/* "$R"/var/cache/apt/*.bin "$R"/var/cache/debconf/*-old "$R"/var/log/* "$R"/tmp/* "$R"/root/.cache # Only needed to compile with pythran, which needs network installs anyway. rm -rf "$R/usr/include/boost" # Docs: keep the copyright/license files, drop the rest. find "$R/usr/share/doc" -mindepth 2 ! -name 'copyright' ! -type d -delete 2>/dev/null || true find "$R/usr/share/doc" -mindepth 1 -type d -empty -delete 2>/dev/null || true rm -rf "$R/usr/share/info" "$R/usr/share/lintian" "$R/usr/share/linda" # Translations and non-English man pages. find "$R/usr/share/locale" -mindepth 1 -maxdepth 1 ! -name 'en*' ! -name 'locale.alias' -exec rm -rf {} + 2>/dev/null || true find "$R/usr/share/man" -mindepth 1 -maxdepth 1 ! -name 'man*' -exec rm -rf {} + 2>/dev/null || true # Test suites shipped inside Python packages. Only directories named "tests": numpy.testing, # sympy.testing and networkx.testing are real modules those libraries import. for p in numpy scipy pandas matplotlib sympy networkx PIL lxml openpyxl; do find "$R/usr/lib/python3/dist-packages/$p" -type d -name tests -prune -exec rm -rf {} + 2>/dev/null || true done fi if should pack; then USED_MB=$(du -sxm "$R" | cut -f1) SIZE_MB=$MAX_MB FREE_EST=$(( SIZE_MB - USED_MB * 110 / 100 )) # ext2 metadata takes roughly 10% log "packing ext2: ${USED_MB} MB used, ${SIZE_MB} MB image, about ${FREE_EST} MB free" if [ "$FREE_EST" -lt "$MIN_FREE_MB" ]; then echo "error: only ~${FREE_EST} MB would be free for the user's files (minimum ${MIN_FREE_MB}); remove packages" >&2 exit 1 fi rm -f "$WORK/disk.ext2" # Same layout as the CheerpX images: ext2, 4 KiB blocks, 256-byte inodes, no reserved blocks. mke2fs -q -t ext2 -b 4096 -I 256 -m 0 -L lwa-debian12 -d "$R" "$WORK/disk.ext2" "${SIZE_MB}M" e2fsck -fn "$WORK/disk.ext2" >/dev/null fi if should chunk; then log "chunking into $REPO/disk" python3 "$HERE/chunk.py" "$WORK/disk.ext2" "$REPO/disk" fi log "done"