"""Interpreter-startup stub for Hugging Face Inference Endpoint containers. Banyan Sapling is a text-only verifier: torchaudio and torchvision are never used. torchaudio has no release matching torch 2.14.x (its final release is 2.11.0), so the container's stale compiled torchaudio extension fails to load against upgraded torch, and transformers 5.x imports torchaudio eagerly while registering models - crashing the worker before the custom handler runs. Pre-importing None into sys.modules here makes `import torchaudio` raise a clean ImportError that transformers treats as "feature unavailable". This is the same mechanism the reference runtime uses (see inference.py: "The text-only runtime does not require torchvision or torchaudio"). Activated by setting PYTHONPATH=/repository on the endpoint, which causes CPython to auto-import this module at interpreter startup (gunicorn master and workers included), before anything imports transformers. """ import sys sys.modules.setdefault("torchaudio", None) sys.modules.setdefault("torchvision", None) def _patch_transformers_file_utils() -> None: """Restore transformers-4.x file_utils symbols that HF's container toolkit still imports (e.g. `from transformers.file_utils import is_tf_available`), which transformers 5.17.0 removed. Predicates answer False - correct for a text-only GPU container (no TF/Flax/TPU/etc.); torch resolves truthily. Applied via a module-level __getattr__ so ANY removed is_*_available name is covered, not just the ones we know about today. """ try: import transformers.file_utils as _fu except Exception: return # transformers not installed yet (e.g. early pip runs) if getattr(_fu, "_banyan_compat_shim", False): return import types as _types class _CompatFileUtils(_types.ModuleType): def __getattr__(self, name): if name.startswith("is_") and name.endswith("_available"): # torch is present (the whole point of this container); # everything else 5.17 removed is genuinely absent here. if name == "is_torch_available": return lambda: True return lambda: False raise AttributeError(f"module {self.__name__!r} has no attribute {name!r}") _fu.__class__ = _CompatFileUtils _fu._banyan_compat_shim = True def _patch_torch_determinism() -> None: """The model's seed_all() enables strict deterministic algorithms; the container's (older) torch raises on ops without deterministic CUDA kernels (e.g. _histc_cuda), killing requests after a successful forward pass. The reference numerics were validated on torch 2.14.1/H200; on this container we keep deterministic mode enabled but with warn_only=True so undeterministic ops warn instead of raising. torch is preinstalled in the container (and in the pre-flight venv), so importing it here is safe; if absent, skip. """ try: import torch except Exception: return if getattr(torch, "_banyan_warn_only_wrap", False): return _orig = torch.use_deterministic_algorithms def _warn_only(mode=True, *, warn_only=False): _orig(bool(mode), warn_only=True) torch.use_deterministic_algorithms = _warn_only torch._banyan_warn_only_wrap = True _patch_transformers_file_utils() _patch_torch_determinism()