axis-numeric / axis_numeric /ort_session.py
serety95's picture
Update numeric space
f1ba785 verified
Raw History Blame Contribute Delete
1.44 kB
"""@module numeric-inference/ort-session — one ORT session, CUDA arena kept small."""
from __future__ import annotations
import os
from .gpu_exec import gpu_call
def open_numeric_session(model_path: str):
import onnxruntime as ort
available = ort.get_available_providers()
use_cuda = "CUDAExecutionProvider" in available
options = ort.SessionOptions()
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
options.intra_op_num_threads = _threads(use_cuda)
options.inter_op_num_threads = 1
options.enable_mem_pattern = True
options.enable_cpu_mem_arena = not use_cuda
providers: list = []
if use_cuda:
providers.append(("CUDAExecutionProvider", {
"arena_extend_strategy": "kSameAsRequested",
"cudnn_conv_algo_search": "HEURISTIC",
"do_copy_in_default_stream": True,
}))
if "CPUExecutionProvider" in available:
providers.append("CPUExecutionProvider")
return gpu_call(lambda: ort.InferenceSession(model_path, sess_options=options, providers=providers or None))
def _threads(use_cuda: bool) -> int:
raw = os.getenv("AXIS_ORT_INTRA_OP", "").strip()
if raw.isdigit() and int(raw) > 0:
return int(raw)
if use_cuda:
return 1
return max(1, min(4, os.cpu_count() or 1))