"""Explicit inference numerics, shared by single/multiple-checkpoint runners.""" import torch def configure_inference(precision): if precision not in ["fp32", "bf16", "tf32"]: raise ValueError("Unknown inference precision") torch.set_num_threads(4) torch.backends.cudnn.benchmark = False torch.backends.cudnn.deterministic = True torch.backends.cudnn.allow_tf32 = precision == "tf32" torch.backends.cuda.matmul.allow_tf32 = precision == "tf32" torch.set_float32_matmul_precision("high" if precision == "tf32" else "highest") def autocast_options(device, precision): cuda = device.startswith("cuda") enabled = cuda and precision == "bf16" if enabled and not torch.cuda.is_bf16_supported(): raise ValueError("BF16 inference requested on an unsupported GPU") return { "device_type": "cuda" if cuda else "cpu", "dtype": torch.bfloat16, "enabled": enabled, }