| """Interpreter-startup stub for Hugging Face Inference Endpoint containers. |
| |
| Banyan Sapling is a text-only verifier: torchaudio and torchvision are never |
| used. torchaudio has no release matching torch 2.14.x (its final release is |
| 2.11.0), so the container's stale compiled torchaudio extension fails to load |
| against upgraded torch, and transformers 5.x imports torchaudio eagerly while |
| registering models - crashing the worker before the custom handler runs. |
| |
| Pre-importing None into sys.modules here makes `import torchaudio` raise a |
| clean ImportError that transformers treats as "feature unavailable". This is |
| the same mechanism the reference runtime uses (see inference.py: "The |
| text-only runtime does not require torchvision or torchaudio"). |
| |
| Activated by setting PYTHONPATH=/repository on the endpoint, which causes |
| CPython to auto-import this module at interpreter startup (gunicorn master |
| and workers included), before anything imports transformers. |
| """ |
| import sys |
|
|
| sys.modules.setdefault("torchaudio", None) |
| sys.modules.setdefault("torchvision", None) |
|
|
|
|
| def _patch_transformers_file_utils() -> None: |
| """Restore transformers-4.x file_utils symbols that HF's container toolkit |
| still imports (e.g. `from transformers.file_utils import is_tf_available`), |
| which transformers 5.17.0 removed. Predicates answer False - correct for a |
| text-only GPU container (no TF/Flax/TPU/etc.); torch resolves truthily. |
| Applied via a module-level __getattr__ so ANY removed is_*_available name |
| is covered, not just the ones we know about today. |
| """ |
| try: |
| import transformers.file_utils as _fu |
| except Exception: |
| return |
| if getattr(_fu, "_banyan_compat_shim", False): |
| return |
| import types as _types |
|
|
| class _CompatFileUtils(_types.ModuleType): |
| def __getattr__(self, name): |
| if name.startswith("is_") and name.endswith("_available"): |
| |
| |
| if name == "is_torch_available": |
| return lambda: True |
| return lambda: False |
| raise AttributeError(f"module {self.__name__!r} has no attribute {name!r}") |
|
|
| _fu.__class__ = _CompatFileUtils |
| _fu._banyan_compat_shim = True |
|
|
|
|
| def _patch_torch_determinism() -> None: |
| """The model's seed_all() enables strict deterministic algorithms; the |
| container's (older) torch raises on ops without deterministic CUDA kernels |
| (e.g. _histc_cuda), killing requests after a successful forward pass. The |
| reference numerics were validated on torch 2.14.1/H200; on this container |
| we keep deterministic mode enabled but with warn_only=True so undeterministic |
| ops warn instead of raising. torch is preinstalled in the container (and in |
| the pre-flight venv), so importing it here is safe; if absent, skip. |
| """ |
| try: |
| import torch |
| except Exception: |
| return |
| if getattr(torch, "_banyan_warn_only_wrap", False): |
| return |
| _orig = torch.use_deterministic_algorithms |
|
|
| def _warn_only(mode=True, *, warn_only=False): |
| _orig(bool(mode), warn_only=True) |
|
|
| torch.use_deterministic_algorithms = _warn_only |
| torch._banyan_warn_only_wrap = True |
|
|
|
|
| _patch_transformers_file_utils() |
| _patch_torch_determinism() |
|
|
