File size: 1,440 Bytes
f1ba785
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
"""@module numeric-inference/ort-session — one ORT session, CUDA arena kept small."""

from __future__ import annotations

import os

from .gpu_exec import gpu_call


def open_numeric_session(model_path: str):
    import onnxruntime as ort

    available = ort.get_available_providers()
    use_cuda = "CUDAExecutionProvider" in available
    options = ort.SessionOptions()
    options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
    options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
    options.intra_op_num_threads = _threads(use_cuda)
    options.inter_op_num_threads = 1
    options.enable_mem_pattern = True
    options.enable_cpu_mem_arena = not use_cuda
    providers: list = []
    if use_cuda:
        providers.append(("CUDAExecutionProvider", {
            "arena_extend_strategy": "kSameAsRequested",
            "cudnn_conv_algo_search": "HEURISTIC",
            "do_copy_in_default_stream": True,
        }))
    if "CPUExecutionProvider" in available:
        providers.append("CPUExecutionProvider")
    return gpu_call(lambda: ort.InferenceSession(model_path, sess_options=options, providers=providers or None))


def _threads(use_cuda: bool) -> int:
    raw = os.getenv("AXIS_ORT_INTRA_OP", "").strip()
    if raw.isdigit() and int(raw) > 0:
        return int(raw)
    if use_cuda:
        return 1
    return max(1, min(4, os.cpu_count() or 1))