Spaces:
Sleeping
Sleeping
File size: 947 Bytes
8fca640 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 | """Explicit inference numerics, shared by single/multiple-checkpoint runners."""
import torch
def configure_inference(precision):
if precision not in ["fp32", "bf16", "tf32"]:
raise ValueError("Unknown inference precision")
torch.set_num_threads(4)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.allow_tf32 = precision == "tf32"
torch.backends.cuda.matmul.allow_tf32 = precision == "tf32"
torch.set_float32_matmul_precision("high" if precision == "tf32" else "highest")
def autocast_options(device, precision):
cuda = device.startswith("cuda")
enabled = cuda and precision == "bf16"
if enabled and not torch.cuda.is_bf16_supported():
raise ValueError("BF16 inference requested on an unsupported GPU")
return {
"device_type": "cuda" if cuda else "cpu",
"dtype": torch.bfloat16,
"enabled": enabled,
}
|