Spaces:
Configuration error
Configuration error
File size: 1,440 Bytes
f1ba785 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 | """@module numeric-inference/ort-session — one ORT session, CUDA arena kept small."""
from __future__ import annotations
import os
from .gpu_exec import gpu_call
def open_numeric_session(model_path: str):
import onnxruntime as ort
available = ort.get_available_providers()
use_cuda = "CUDAExecutionProvider" in available
options = ort.SessionOptions()
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
options.intra_op_num_threads = _threads(use_cuda)
options.inter_op_num_threads = 1
options.enable_mem_pattern = True
options.enable_cpu_mem_arena = not use_cuda
providers: list = []
if use_cuda:
providers.append(("CUDAExecutionProvider", {
"arena_extend_strategy": "kSameAsRequested",
"cudnn_conv_algo_search": "HEURISTIC",
"do_copy_in_default_stream": True,
}))
if "CPUExecutionProvider" in available:
providers.append("CPUExecutionProvider")
return gpu_call(lambda: ort.InferenceSession(model_path, sess_options=options, providers=providers or None))
def _threads(use_cuda: bool) -> int:
raw = os.getenv("AXIS_ORT_INTRA_OP", "").strip()
if raw.isdigit() and int(raw) > 0:
return int(raw)
if use_cuda:
return 1
return max(1, min(4, os.cpu_count() or 1))
|