LocalWebAgent / image /build.sh
ereniko's picture
v0.9.0: Debian 12 Linux image with a much bigger toolset
0d56bc3 unverified
Raw History Blame Contribute Delete
8.3 kB
#!/bin/bash
# Builds LocalWebAgent's Linux disk: a Debian 12 (bookworm) i386 ext2 image for CheerpX,
# split into the 4 MiB gzipped chunks + manifest.json the app downloads (see chunk.py).
#
# sudo image/build.sh [workdir] full build into workdir (default /tmp/lwa-image)
# sudo image/build.sh [workdir] --from customize rerun from a stage (bootstrap, install,
# customize, slim, pack, chunk)
#
# Needs: root, debootstrap, e2fsprogs, python3, and an x86 host that can run i386 binaries.
# Behind an HTTPS proxy, set https_proxy and optionally BUILD_CA=/path/to/proxy-ca.pem;
# both are only used during the build and removed from the image.
set -euo pipefail
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO="$(dirname "$HERE")"
WORK="${1:-/tmp/lwa-image}"
FROM="${3:-bootstrap}"
[ "${2:-}" = "--from" ] || FROM=bootstrap
R="$WORK/rootfs"
MIRROR="${MIRROR:-https://deb.debian.org/debian}"
# CheerpX's HttpBytesDevice refuses disks over 2 GiB (cx_esm.js checks Content-Range against
# 2147483648), so the whole filesystem, software plus free space, must fit in that.
MAX_MB=2048
MIN_FREE_MB="${MIN_FREE_MB:-450}" # fail the build if fewer MB would be left for the user's files
STAGES=(bootstrap install customize slim pack chunk)
log() { printf '\033[1;32m==> %s\033[0m\n' "$*"; }
run_in() { chroot "$R" /usr/bin/env -i PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
HOME=/root LANG=C.UTF-8 DEBIAN_FRONTEND=noninteractive "$@"; }
mounts() {
for m in proc sys dev dev/pts; do mountpoint -q "$R/$m" || mount --bind "/$m" "$R/$m"; done
}
umounts() {
for m in dev/pts dev sys proc; do mountpoint -q "$R/$m" && umount -l "$R/$m" || true; done
}
trap umounts EXIT
should() { # should <stage>: true if <stage> runs given --from
local s seen=0
for s in "${STAGES[@]}"; do [ "$s" = "$FROM" ] && seen=1; [ "$s" = "$1" ] && return $((1 - seen)); done
return 1
}
mkdir -p "$WORK"
if should bootstrap; then
log "debootstrap Debian 12 i386"
rm -rf "$R"
debootstrap --arch=i386 --variant=minbase bookworm "$R" "$MIRROR"
fi
if should install; then
log "apt configuration (build-only proxy/CA settings are removed in 'slim')"
cat > "$R/etc/apt/sources.list" <<EOF
deb $MIRROR bookworm main
deb $MIRROR bookworm-updates main
deb ${MIRROR%/debian}/debian-security bookworm-security main
EOF
{
echo 'APT::Install-Recommends "false";'
echo 'APT::Install-Suggests "false";'
if [ -n "${https_proxy:-}" ]; then
echo "Acquire::http::Proxy \"$https_proxy\";"
echo "Acquire::https::Proxy \"$https_proxy\";"
fi
if [ -n "${BUILD_CA:-}" ]; then
cp "$BUILD_CA" "$R/etc/apt/lwa-build-ca.crt"
echo 'Acquire::https::CAInfo "/etc/apt/lwa-build-ca.crt";'
fi
} > "$R/etc/apt/apt.conf.d/99lwa-build"
cp /etc/resolv.conf "$R/etc/resolv.conf"
mounts
log "installing packages ($(grep -v '^#' "$HERE/packages.txt" | wc -w))"
run_in apt-get update -q
run_in apt-get -y -q dist-upgrade
# shellcheck disable=SC2046
run_in apt-get install -y -q $(grep -v '^#' "$HERE/packages.txt")
umounts
fi
if should customize; then
log "customizing"
mounts
# The VM runs as uid 1000 "user" (see js/vm.js).
run_in id user >/dev/null 2>&1 || run_in useradd -m -u 1000 -s /bin/bash user
cp -a "$HERE/overlay/." "$R/"
run_in chown -R 1000:1000 /home/user
echo localwebagent > "$R/etc/hostname"
# Friendlier names for Debian's renamed tools.
ln -sf /usr/bin/python3 "$R/usr/local/bin/python"
ln -sf /usr/bin/fdfind "$R/usr/local/bin/fd"
ln -sf /usr/bin/batcat "$R/usr/local/bin/bat"
# Environment for the VM's shell (interactive, non-login: it reads /etc/bash.bashrc,
# not /etc/profile.d). Offline Go: never try to download modules.
sed -i '/^# --- LocalWebAgent ---$/,$d' "$R/etc/bash.bashrc"
cat >> "$R/etc/bash.bashrc" <<'EOF'
# --- LocalWebAgent ---
export MPLBACKEND=Agg
# Single-threaded math libraries: thread pools can deadlock on futexes under CheerpX.
export OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 MKL_NUM_THREADS=1 NUMEXPR_MAX_THREADS=1
alias ll='ls -alF --color=auto' la='ls -A --color=auto' l='ls -CF --color=auto'
EOF
# numpy/scipy: use the reference BLAS/LAPACK, not OpenBLAS. OpenBLAS starts a thread pool
# that deadlocks at exit under CheerpX and prints an L2-cache warning on every import.
# (Its packages stay installed: scipy -> pythran depends on them.)
run_in update-alternatives --set libblas.so.3-i386-linux-gnu /usr/lib/i386-linux-gnu/blas/libblas.so.3
run_in update-alternatives --set liblapack.so.3-i386-linux-gnu /usr/lib/i386-linux-gnu/lapack/liblapack.so.3
printf 'OPENBLAS_NUM_THREADS=1\nOMP_NUM_THREADS=1\n' > "$R/etc/environment"
# Threading fix for CheerpX (see compat/lwa-compat.c), preloaded into every process.
cp "$HERE/compat/lwa-compat.c" "$R/tmp/lwa-compat.c"
run_in gcc -O2 -Wall -shared -fPIC -o /usr/local/lib/liblwa-compat.so /tmp/lwa-compat.c -pthread
echo /usr/local/lib/liblwa-compat.so > "$R/etc/ld.so.preload"
cat > "$R/tmp/join.c" <<'EOF'
#include <pthread.h>
#include <stdio.h>
#include <unistd.h>
static void *slow(void *a) { usleep(50000); return a; }
int main(void) { pthread_t t; pthread_create(&t, 0, slow, (void *)42); void *r; int e = pthread_join(t, &r); printf("%d %ld\n", e, (long)r); return e; }
EOF
run_in gcc -pthread -o /tmp/join /tmp/join.c
[ "$(run_in /tmp/join)" = "0 42" ] || { echo "lwa-compat self-test failed" >&2; exit 1; }
rm -f "$R"/tmp/join "$R"/tmp/join.c "$R"/tmp/lwa-compat.c
# Precompile Python bytecode so the first import doesn't write to the user's overlay.
run_in python3 -m compileall -q -j 0 /usr/lib/python3/dist-packages /usr/lib/python3.11 >/dev/null || true
# Font cache for matplotlib / graphviz, built once here.
run_in fc-cache -f >/dev/null 2>&1 || true
# Matplotlib's font cache, built where the VM user's matplotlib looks for it.
chroot --userspec=1000:1000 "$R" /usr/bin/env -i HOME=/home/user PATH=/usr/bin:/bin MPLBACKEND=Agg \
python3 -c "import matplotlib.pyplot" || true
umounts
fi
if should slim; then
log "slimming"
rm -f "$R/etc/apt/apt.conf.d/99lwa-build" "$R/etc/apt/lwa-build-ca.crt" "$R/etc/resolv.conf"
: > "$R/etc/machine-id"
run_in apt-get clean
rm -rf "$R"/var/lib/apt/lists/* "$R"/var/cache/apt/*.bin "$R"/var/cache/debconf/*-old "$R"/var/log/* "$R"/tmp/* "$R"/root/.cache
# Only needed to compile with pythran, which needs network installs anyway.
rm -rf "$R/usr/include/boost"
# Docs: keep the copyright/license files, drop the rest.
find "$R/usr/share/doc" -mindepth 2 ! -name 'copyright' ! -type d -delete 2>/dev/null || true
find "$R/usr/share/doc" -mindepth 1 -type d -empty -delete 2>/dev/null || true
rm -rf "$R/usr/share/info" "$R/usr/share/lintian" "$R/usr/share/linda"
# Translations and non-English man pages.
find "$R/usr/share/locale" -mindepth 1 -maxdepth 1 ! -name 'en*' ! -name 'locale.alias' -exec rm -rf {} + 2>/dev/null || true
find "$R/usr/share/man" -mindepth 1 -maxdepth 1 ! -name 'man*' -exec rm -rf {} + 2>/dev/null || true
# Test suites shipped inside Python packages. Only directories named "tests": numpy.testing,
# sympy.testing and networkx.testing are real modules those libraries import.
for p in numpy scipy pandas matplotlib sympy networkx PIL lxml openpyxl; do
find "$R/usr/lib/python3/dist-packages/$p" -type d -name tests -prune -exec rm -rf {} + 2>/dev/null || true
done
fi
if should pack; then
USED_MB=$(du -sxm "$R" | cut -f1)
SIZE_MB=$MAX_MB
FREE_EST=$(( SIZE_MB - USED_MB * 110 / 100 )) # ext2 metadata takes roughly 10%
log "packing ext2: ${USED_MB} MB used, ${SIZE_MB} MB image, about ${FREE_EST} MB free"
if [ "$FREE_EST" -lt "$MIN_FREE_MB" ]; then
echo "error: only ~${FREE_EST} MB would be free for the user's files (minimum ${MIN_FREE_MB}); remove packages" >&2
exit 1
fi
rm -f "$WORK/disk.ext2"
# Same layout as the CheerpX images: ext2, 4 KiB blocks, 256-byte inodes, no reserved blocks.
mke2fs -q -t ext2 -b 4096 -I 256 -m 0 -L lwa-debian12 -d "$R" "$WORK/disk.ext2" "${SIZE_MB}M"
e2fsck -fn "$WORK/disk.ext2" >/dev/null
fi
if should chunk; then
log "chunking into $REPO/disk"
python3 "$HERE/chunk.py" "$WORK/disk.ext2" "$REPO/disk"
fi
log "done"