scripts/nvidia_lib_shim.sh

#!/usr/bin/env bash
# Fixes "Failed to initialize NVML: Driver/library version mismatch", which
# happens on this host because the kernel module (570.172.08) is older than
# the installed userspace libraries (580.173.02). No sudo/reboot needed: the
# matching 570.172.08 .so files are still on disk at
# /usr/lib/x86_64-linux-gnu -- the dynamic linker just isn't picking them by
# default -- so this only has to make them findable first via LD_LIBRARY_PATH.
#
# Usage:
#   source scripts/nvidia_lib_shim.sh          # sets LD_LIBRARY_PATH in this shell
#   ./scripts/nvidia_lib_shim.sh nvidia-smi    # runs one command with the shim applied
#
# Deliberately not `set -e`: this script must not kill the caller's shell
# when sourced, even on a benign miss (e.g. re-running after the driver was
# already fixed properly).
set -uo pipefail

nvshim_driver_version="570.172.08"
nvshim_lib_dir="/usr/lib/x86_64-linux-gnu"
nvshim_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)"
nvshim_repo_root="$(cd "${nvshim_script_dir}/.." && pwd)"
nvshim_dir="${nvshim_repo_root}/.nvshim"

# --- Refuse to shim once the host no longer needs it ------------------------
#
# MEASURED 2026-08-06, and the reason this guard exists: the host's driver was
# upgraded so that kernel and userspace now BOTH report 580.173.02 -- there is
# no mismatch left to fix. The 570.172.08 files this script links to were
# truncated to **0 bytes** by that upgrade (`ls -la` confirms size 0, dated
# Jun 19). Linking them and putting the directory first on LD_LIBRARY_PATH
# therefore does not "fix" NVML, it destroys it:
#
#     LD_LIBRARY_PATH=.nvshim nvidia-smi
#     -> NVIDIA-SMI couldn't find libnvidia-ml.so library in your system
#
# and every CUDA process launched from that shell inherits the breakage. This
# script had already been disabled once for exactly this reason (the directory
# was renamed .nvshim.disabled-driver580), but the README still tells the
# reader to source it, so it kept getting re-enabled by anyone following the
# instructions -- recreating .nvshim as a side effect of merely running it.
#
# So the guard lives here, in the script itself, rather than in documentation:
# a stale-shim trap that a person or agent can walk into by following the
# repo's own README is not a documentation problem.
nvshim_kernel_version="$(cat /proc/driver/nvidia/version 2>/dev/null \
    | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1)"
if [ -n "${nvshim_kernel_version}" ] && [ "${nvshim_kernel_version}" != "${nvshim_driver_version}" ]; then
    echo "nvidia_lib_shim: kernel driver is ${nvshim_kernel_version}, not ${nvshim_driver_version}." >&2
    echo "nvidia_lib_shim: no mismatch to shim -- doing nothing (LD_LIBRARY_PATH untouched)." >&2
    echo "nvidia_lib_shim: forcing it anyway would put 0-byte ${nvshim_driver_version} stubs" >&2
    echo "nvidia_lib_shim: ahead of the working libraries and break NVML for this shell." >&2
    rm -rf "${nvshim_dir}"
    return 0 2>/dev/null || exit 0
fi

mkdir -p "${nvshim_dir}"

nvshim_n_linked=0
for nvshim_lib in "${nvshim_lib_dir}"/*."${nvshim_driver_version}"; do
    [ -e "${nvshim_lib}" ] || continue
    # A truncated library is worse than a missing one: it resolves, then fails
    # at the first symbol lookup, far from here.
    [ -s "${nvshim_lib}" ] || continue
    nvshim_base="$(basename "${nvshim_lib}")"
    nvshim_stem="${nvshim_base%."${nvshim_driver_version}"}"  # libnvidia-ml.so.570.172.08 -> libnvidia-ml.so
    ln -sf "${nvshim_lib}" "${nvshim_dir}/${nvshim_stem}"      # libfoo.so
    ln -sf "${nvshim_lib}" "${nvshim_dir}/${nvshim_stem}.1"    # libfoo.so.1 (the SONAME most loaders resolve)
    nvshim_n_linked=$((nvshim_n_linked + 1))
done

if [ "${nvshim_n_linked}" -eq 0 ]; then
    echo "nvidia_lib_shim: no *.${nvshim_driver_version} libraries found under ${nvshim_lib_dir}" >&2
    nvshim_status=1
else
    export LD_LIBRARY_PATH="${nvshim_dir}:${LD_LIBRARY_PATH:-}"
    echo "nvidia_lib_shim: shimmed ${nvshim_n_linked} libraries -> ${nvshim_dir}" >&2
    echo "nvidia_lib_shim: LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >&2
    nvshim_status=0
fi

unset nvshim_driver_version nvshim_lib_dir nvshim_script_dir nvshim_dir
unset nvshim_lib nvshim_base nvshim_stem nvshim_n_linked nvshim_repo_root

if [ "$#" -gt 0 ]; then
    if [ "${nvshim_status}" -ne 0 ]; then
        unset nvshim_status
        exit 1
    fi
    unset nvshim_status
    exec "$@"
fi
unset nvshim_status