farrell236's picture
Deploy native Gradio OCT inference app
8fca640 verified
Raw History Blame Contribute Delete
947 Bytes
"""Explicit inference numerics, shared by single/multiple-checkpoint runners."""
import torch
def configure_inference(precision):
if precision not in ["fp32", "bf16", "tf32"]:
raise ValueError("Unknown inference precision")
torch.set_num_threads(4)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.allow_tf32 = precision == "tf32"
torch.backends.cuda.matmul.allow_tf32 = precision == "tf32"
torch.set_float32_matmul_precision("high" if precision == "tf32" else "highest")
def autocast_options(device, precision):
cuda = device.startswith("cuda")
enabled = cuda and precision == "bf16"
if enabled and not torch.cuda.is_bf16_supported():
raise ValueError("BF16 inference requested on an unsupported GPU")
return {
"device_type": "cuda" if cuda else "cpu",
"dtype": torch.bfloat16,
"enabled": enabled,
}