File size: 947 Bytes
8fca640
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
"""Explicit inference numerics, shared by single/multiple-checkpoint runners."""

import torch


def configure_inference(precision):
    if precision not in ["fp32", "bf16", "tf32"]:
        raise ValueError("Unknown inference precision")
    torch.set_num_threads(4)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.allow_tf32 = precision == "tf32"
    torch.backends.cuda.matmul.allow_tf32 = precision == "tf32"
    torch.set_float32_matmul_precision("high" if precision == "tf32" else "highest")


def autocast_options(device, precision):
    cuda = device.startswith("cuda")
    enabled = cuda and precision == "bf16"
    if enabled and not torch.cuda.is_bf16_supported():
        raise ValueError("BF16 inference requested on an unsupported GPU")
    return {
        "device_type": "cuda" if cuda else "cpu",
        "dtype": torch.bfloat16,
        "enabled": enabled,
    }