"""Bounded asynchronous NPZ serialization; values and dtypes are preserved.""" from collections import deque from concurrent.futures import ThreadPoolExecutor from pathlib import Path import zipfile import numpy as np def save_npz(path, arrays, level=1): path = Path(path) temporary = path.with_suffix(".tmp.npz") with zipfile.ZipFile( temporary, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=level, allowZip64=True, ) as archive: for name, value in arrays.items(): with archive.open(name + ".npy", "w", force_zip64=True) as handle: np.lib.format.write_array(handle, np.asarray(value), allow_pickle=False) temporary.replace(path) class PredictionWriter: def __init__(self, output_dir, workers=2, maximum_pending_cases=2): self.output_dir = Path(output_dir) self.queue = deque() self.maximum_pending_cases = maximum_pending_cases self.executor = ThreadPoolExecutor(max_workers=workers) def submit(self, case_id, items): # Keep at most this many completed-volume arrays alive behind inference. while len(self.queue) >= self.maximum_pending_cases: self.queue.popleft().result() def write_case(): for name, positions, keys in items: save_npz( self.output_dir / name / (case_id + ".npz"), {"positions": positions, "boundary_keys": np.asarray(keys)}, ) self.queue.append(self.executor.submit(write_case)) def close(self): try: while self.queue: self.queue.popleft().result() finally: self.executor.shutdown(wait=True, cancel_futures=True)